def job():
    logger.info("Getting page")
    txt = yield client.getPage(settings.url)
    parsed_txt = html.fromstring(txt)
    # pylint:disable=E1102
    el = sel_base(parsed_txt)
    # if element does not exist
    if not len(el):
        return
    chapter_name = sel_chapter_name(el[0])[0].text.strip()
    update_date = sel_update_date(el[0])[0].text.strip()

    # do nothing if no new chapter avaliable
    if settings.current_chapter == chapter_name:
        logger.info("Current chapter is %s, not a new chapter, do nothing.", chapter_name)
        return
    else:
        # saves the new chapter name to settings
        settings.current_chapter = chapter_name

    data = {
        "chapter_name": chapter_name,
        "update_date": update_date
    }
    logger.info("{\n\tchapter_name: %s,\n\tupdate_date: %s\n}", chapter_name, update_date)

    logger.info("Sending data to webhook: %s", settings.webhook_url)
    try:
        yield client.getPage(settings.webhook_url,
                            method="POST", postdata=json.dumps(data),
                            headers={"Content-Type": "application/json; chatset=utf-8"})
    except Exception as exc:
        logger.error("Error occured while sending data.\n%s", str(exc))
    else:
        logger.info("Data sent")

		def getFromXunlei(result,parameter):
			header,tailer,filename = parameter
			url = 'http://bt.box.n0808.com/%s/%s/%s' % (header,tailer,filename)
			self.transport.write('start downloading from %s\n' % url)
			getPage(url,filename).addCallbacks(
				ParseTorrent,
				lambda result:self.transport.write('download torrent failed\n'))

	def startRun(self):
		if self.weatherPluginEntry is not None:
			self["statustext"].text = _("Getting weather information...")
			url = "http://weather.service.msn.com/data.aspx?weadegreetype=%s&culture=%s&wealocations=%s" % (self.weatherPluginEntry.degreetype.value, self.language, self.weatherPluginEntry.weatherlocationcode.value)
			getPage(url).addCallback(self.xmlCallback).addErrback(self.error)
		else:
			self["statustext"].text = _("No locations defined...\nPress 'Menu' to do that.")

    def openTest(self):
                self["info"].setText("Downloading list...")
                testno = 1

                xurl = "http://www.et-view-support.com/addons/XTA-team/" + self.addon + "/list.txt"
                print "xurl =", xurl
                getPage(xurl).addCallback(self.gotPage).addErrback(self.getfeedError)

	def monsteruploads(self, data, url):
		if not re.match('.*?eval\(function\(p\,a\,c\,k\,e\,d', data, re.S):
			id = re.findall('type="hidden" name="id".*?value="(.*?)"', data, re.S)
			fname = re.findall('type="hidden" name="fname".*?value="(.*?)"', data, re.S)
			referer = re.findall('type="hidden" name="referer".*?value="(.*?)"', data, re.S)
			info = urlencode({
				'op':           "download2",
				'usr_login':    "",
				'id':           id[0],
				'fname':        fname[0],
				'referer':      "",
				'method_free':  "Kostenloser Download"
				})
			getPage(url, method='POST', postdata = info, headers={'Content-Type':'application/x-www-form-urlencoded'}).addCallback(self.monsteruploads_post).addErrback(self.errorload)
			
		else:
		
			get_packedjava = re.findall("<script type=.text.javascript.>eval.function(.*?)</script>", data, re.S|re.DOTALL)
			if get_packedjava:
				sJavascript = get_packedjava[1]
				sUnpacked = cJsUnpacker().unpackByString(sJavascript)
				if sUnpacked:
					stream_url = re.findall('type="video/divx"src="(.*?)"', sUnpacked)
					if stream_url:
						print stream_url[0]
						self._callback(stream_url[0])		
					else:
						self.stream_not_found()
				else:
					self.stream_not_found()
			else:
				self.stream_not_found()

	def getCookie(self, data=None):
		#url = "http://www.skygo.sky.de/index.html"
		url = "http://www.skygo.sky.de/skyguide"
		msg_log = "get page: '{0}'".format(url)
		print msg_log
		self.addLog(msg_log)
		getPage(url, headers=self.headers1, agent=self.agent, timeout=30, cookies=self.ck).addCallback(self.getSkyGuidePages).addErrback(self.dataError)

	def	request(self):
		# if self.serien_name in self.termineCache:
		# 	self.__callback(self.termineCache[self.serien_name], False)
		# else:
		print "[SerienRecorder] suche ' %s '" % self.serien_name
		print self.serie_url
		getPage(getURLWithProxy(self.serie_url), agent=getUserAgent(), headers=getHeaders()).addCallback(self.__callback).addErrback(self.__errback)

 def recaptcha(self, data):
     if data.find('http://goo.gl') != -1:
         self.mod.on_recaptcha(self.index, data[data.index('"id"'):].split('"')[3])
     else:
         self.challenge = data[data.index('challenge'):].split('\'')[1]
         url = '{"longUrl": "http://www.google.com/recaptcha/api/image?c=%s"}' % self.challenge
         getPage("https://www.googleapis.com/urlshortener/v1/url", method="POST", postdata=url, headers={'Content-Type': 'application/json'}).addCallback(self.recaptcha)

    def start(self):
        try:
            data = yield getPage("%slogin/" % self.config['url'], cookies=self.cookies)
        except Exception:
            raise

        result = CSRF_RE.search(data)
        if not result:
            print "error"
            return
        token = result.group(1)

        # login
        try:
            data = yield getPage("%slogin/" % self.config['url'],
                                 method='POST',
                                 postdata=urllib.urlencode({
                                     'csrfmiddlewaretoken': token,
                                     'username': self.config['username'],
                                     'password': self.config['password'],
                                 }),
                                 cookies=self.cookies,
                                 headers={'Content-Type': 'application/x-www-form-urlencoded'})
        except Exception:
            raise

        self.task = task.LoopingCall(self.update)
        self.task.start(self.config["update_rate"])

    def updateTimer(self):
        reactor.callLater(UPDATETIMER, self.updateTimer)

        def setTreeStatus(result):
            r = json.loads(result)
            status = r[u"status"]
            treename = r[u"tree"]
            changed = False
            logmsg = "Regular Tree check says: {} is {}".format(treename, status)
            if not treename in self.statusCache:
                changed = True
            if treename in self.statusCache and status != self.statusCache[treename]:
                changed = True
            if changed:
                # if status previously unknown (= bot has just started)
                # or a changed status then set the topic
                channel = tree2channel[tree]
                topic = "{} is {}!".format(treename, status)
                self.topic(channel, topic)
                logmsg += ", which is new. changing topic."
                print logmsg
            else:
                logmsg += ", just like before"
                print logmsg
            self.statusCache[treename] = status

        for tree in tree2channel:
            url = URL.format(tree)
            getPage(url).addCallbacks(callback=setTreeStatus)

    def test_start_stop(self):
        yield self._storage.put("abc", StringIO("hello world"))
        yield self._server.start()
        # Starting multiple times is fine.
        yield self._server.start()
        storage_url = yield self._storage.get_url("abc")

        # It might not have started actually accepting connections yet...
        yield self.wait_for_server(self._server)
        self.assertEqual((yield getPage(storage_url)), "hello world")

        # Check that it can be killed by the current user (ie, is not running
        # as root) and still comes back up
        old_pid = yield self._server.get_pid()
        os.kill(old_pid, signal.SIGKILL)
        new_pid = yield self._server.get_pid()
        self.assertNotEquals(old_pid, new_pid)

        # Give it a moment to actually start serving again
        yield self.wait_for_server(self._server)
        self.assertEqual((yield getPage(storage_url)), "hello world")

        yield self._server.stop()
        # Stopping multiple times is fine too.
        yield self._server.stop()

    def delete(self):
        """
        Delete the specific workflow from VelesForge. Requires write access.
        """
        if not self._confirm():
            return
        print("Deleting %s..." % self.name)
        sys.stdout.flush()

        def finished(response):
            print("Successfully deleted %s" % self.name)
            sys.stdout.flush()
            self.stop()

        def failed(failure):
            try:
                failure.raiseException()
            except:
                self.exception("Failed to delete %s:", self.name)
            self.stop(failure, False)

        url = "%s%s?query=delete&name=%s&token=%s" % (
            self.base, root.common.forge.service_name, self.name, self.id)
        self.debug("Requesting %s", url)
        getPage(url.encode('charmap')).addCallbacks(callback=finished,
                                                    errback=failed)

    def list(self):
        """
        Print the list of available workflows on VelesForge.
        """
        def finished(response):
            self.debug("Received %s", response)
            try:
                response = json.loads(response.decode('UTF-8'))
            except ValueError as e:
                self.exception("Failed to parse the response from server: %s",
                               response)
                self.stop(Failure(e), False)
                return
            table = PrettyTable("Name", "Description", "Author", "Version",
                                "Date")
            table.align["Name"] = table.align["Description"] = 'l'
            for item in response:
                table.add_row(*item)
            print(table)
            sys.stdout.flush()
            self.stop()

        def failed(failure):
            self.return_code = 1
            try:
                failure.raiseException()
            except:
                self.exception("Failed to list the available models:")
            self.stop(failure, False)

        url = self.base + root.common.forge.service_name + "?query=list"
        self.debug("Requesting %s", url)
        getPage(url.encode('charmap')).addCallbacks(callback=finished,
                                                    errback=failed)

    def __init__(self, session):
        Screen.__init__(self, session)
        self["info"] = Label('Ładuję...')
        self["myPic"] = Pixmap()
        try:
            os.unlink('/tmp/meteo.png')
        except:
            pass
        self["myActionMap"] = ActionMap(["MeteoActions"],
        {
            "ok": self.cancel,
            "cancel": self.cancel,
            "menu": self.menu,
            "blue": self.menu,
            "green": self.next,
            "left": self.prev,
            "right": self.next,
            "start": self.start,
            "info": self.ShowOpis,
            "yellow": self.ShowOpis,
            "list": self.choose,
        }, -1)

        getPage('http://e2.areq.eu.org/ummeteo/version').addCallback(self.updateCB).addErrback(self.errorUpdate)

        self.onLayoutFinish.append(self.layoutFinished)

	def handleWebsite(self, website):
		info(website.getAttribute("name"))
		if self.number[:2] == "00":
			number = website.getAttribute("prefix") + self.number.replace(self.countrycode, "")
		else:
			number = self.number

		url = website.getAttribute("url")
		if re.search('$AREACODE', url) or re.search('$PFXAREACODE', url):
			error("(PFX)ARECODE cannot be handled")
			# self.caller = _("UNKNOWN")
			self.notifyAndReset()
			return
		#
		# Apparently, there is no attribute called (pfx)areacode anymore
		# So, this below will not work.
		#
		if re.search('\\$AREACODE', url) and website.hasAttribute("areacode"):
			areaCodeLen = int(website.getAttribute("areacode"))
			url = url.replace("$AREACODE", number[:areaCodeLen]).replace("$NUMBER", number[areaCodeLen:])
		elif re.search('\\$PFXAREACODE', url) and website.hasAttribute("pfxareacode"):
			areaCodeLen = int(website.getAttribute("pfxareacode"))
			url = url.replace("$PFXAREACODE", "%(pfxareacode)s").replace("$NUMBER", "%(number)s")
			url = url % {'pfxareacode': number[:areaCodeLen], 'number': number[areaCodeLen:]}
		elif re.search('\\$NUMBER', url):
			url = url.replace("$NUMBER", "%s") % number
		else:
			error("cannot handle websites with no $NUMBER in url")
			# self.caller = _("UNKNOWN")
			self.notifyAndReset()
			return
		info("Url to query: " + url)
		url = url.encode("UTF-8", "replace")
		self.currentWebsite = website
		getPage(url, agent="Mozilla/5.0 (Windows; U; Windows NT 6.0; de; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5").addCallback(self._gotPage).addErrback(self._gotError)

 def start(self):
     # TODO: recheck listening interface; may need to rebind manager to renewed IP
     if 'state_changed_alert' in self.dfm:
         raise Exception('An operation is already in progress')
     self.dfm['state_changed_alert'] = defer.Deferred()
     if not self.torrent and not self.download.metadata:
         if self.download.url:
             # Don't try to re-fetch until an attempt is done
             if self.download.status != Status.LOADING:
                 if self.download.url.startswith('magnet:'):
                     params = { 'save_path': str(self.directory), 'auto_managed': True }
                     resdata = None
                     if self.download.resume_data:
                         params['resdata'] = marshal.loads(self.download.resume_data)
                     self.download.status = Status.LOADING
                     self.torrent = lt_manager.add_magnet(self, str(self.download.url), params)
                     self.dfm['metadata_received_alert'] = defer.Deferred()
                     self.dfm['metadata_received_alert'].addCallback(self.magnet_loaded)
                     self.dfm['metadata_failed_alert'] = defer.Deferred()
                     self.dfm['metadata_failed_alert'].addCallback(self.magnet_load_failed)
                     return self.dfm['state_changed_alert']
                 else:
                     self.download.status = Status.LOADING
                     self.download.status_message = u'Getting the torrent metadata'
                     client.getPage(str(self.download.url)).addCallback(self.fetch_torrent_success).addErrback(self.fetch_torrent_failure)
             return self.dfm['state_changed_alert']
         else:
             raise Exception('Torrent metadata missing and no source URL specified')
     else:
         return self.start_real()

def _do_fetch_gitrev(build_url, base_url, sourcetarball_url, platform, logger):
    result = {}

    # get git revision from jenkins
    json_all = json.loads((yield _mk_request_jenkins_all_builds(base_url, platform, logger)))
    gitrev = None
    for i in [i for i in json_all['allBuilds'] if i['url'].encode('ascii', 'ignore') == build_url]:
        for fingerprint in i['fingerprint']:
            fp_name = fingerprint['original']['name']
            if fp_name == 'build-source-tarball-mono':
                assert gitrev is None, "should set gitrev only once"
                url = sourcetarball_url + str(fingerprint['original']['number']) + '/pollingLog/pollingLog'
                if logger:
                    logger("request: " + str(url))
                regexgitrev = re.compile("Latest remote head revision on [a-zA-Z/]+ is: (?P<gitrev>[0-9a-fA-F]+)")
                match = regexgitrev.search((yield getPage(url)))
                assert match is not None
                gitrev = match.group('gitrev')
            if fp_name == 'build-source-tarball-mono-pullrequest':
                assert gitrev is None, "should set gitrev only once"
                url = sourcetarball_url + str(fingerprint['original']['number']) + '/consoleText'
                if logger:
                    logger("request: " + str(url))
                regexgitrev = re.compile("GitHub pull request #(?P<prid>[0-9]+) of commit (?P<gitrev>[0-9a-fA-F]+)")
                match = regexgitrev.search((yield getPage(url)))
                assert match is not None
                gitrev = match.group('gitrev')
                result[PROPERTYNAME_JENKINSGITHUBPULLREQUEST] = match.group('prid')
    assert gitrev is not None, "parsing gitrev failed"
    result[PROPERTYNAME_JENKINSGITCOMMIT] = gitrev
    defer.returnValue(result)

	def GetWeatherInfo(self):
		self.url = "http://www.homeys-bunker.de/dm800/feeds/weatherinfo2.php?citycode=%s&metric=%s" % (config.plugins.EVOMC_wi.city.value, config.plugins.EVOMC_wi.metric.value)
		try:
			getPage(self.url).addCallback(self.UpdateWeatherInfo).addErrback(self.error)
			self["CurrentCity"].setText(_("Updating Weather Info ..."))
		except Exception, e:
			self["CurrentCity"].setText(_("Updating Weather Info failed (Twisted Web not installed)"))

	def eco_post(self, data):
		url = "http://www.ecostream.tv/assets/js/common.js"
		data2 = urllib.urlopen(url).read()
		post_url = re.findall("url: '(http://www.ecostream.tv/.*?)\?s=", data2, re.S)
		if post_url:
			print post_url
			sPattern = "var t=setTimeout\(\"lc\('([^']+)','([^']+)','([^']+)','([^']+)'\)"
			r = re.findall(sPattern, data)
			if r:
				for aEntry in r:
					sS = str(aEntry[0])
					sK = str(aEntry[1])
					sT = str(aEntry[2])
					sKey = str(aEntry[3])

				print "current keys:", sS, sK, sT, sKey
				sNextUrl = post_url[0]+"?s="+sS+'&k='+sK+'&t='+sT+'&key='+sKey
				print "URL:", sNextUrl
				info = urlencode({'s': sS, 'k': sK, 't': sT, 'key': sKey})
				print "POST:", info
				getPage(sNextUrl, method='POST', postdata=info, headers={'Referer':'http://www.ecostream.tv', 'X-Requested-With':'XMLHttpRequest'}).addCallback(self.eco_final).addErrback(self.errorload)
			else:
				self.stream_not_found()
		else:
			self.stream_not_found()

def do_fetch(url):
    subscribers = len(subscriptions.get(url, []))
    if subscribers:
        print "Fetching %s for %s subscribers" % (url, subscribers)
        client.getPage(url, headers={'X-Hub-Subscribers': subscribers}) \
            .addCallbacks(lambda p: dispatch_queue.put({'topic': url, 'data': p, 'content_type': 'application/atom+xml'}))
    fetch_queue.get().addCallback(do_fetch)