def slurp_with_login_and_pwd():
    import sys
    import mechanize
    # sys.path.append('ClientCookie-1.0.3')
    # from mechanize import ClientCookie
    # sys.path.append('ClientForm-0.1.17')
    # import ClientForm

    # Create special URL opener (for User-Agent) and cookieJar
    cookieJar = mechanize.CookieJar()

    opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [("User-agent","Mozilla/5.0 (compatible)")]
    mechanize.install_opener(opener)
    fp = mechanize.urlopen("http://login.yahoo.com")
    forms = mechanize.ParseResponse(fp)
    fp.close()

    # print forms on this page
    for form in forms:
        print "***************************"
        print form

    form = forms[0]
    form["login"]  = "yahoo-user-id" # use your userid
    form["passwd"] = "password"      # use your password
    fp = mechanize.urlopen(form.click())
    fp.close()
    fp = mechanize.urlopen("https://class.coursera.org/ml-003/lecture/download.mp4?lecture_id=1") # use your group
    fp.readlines()
    fp.close()

    def main(self):
        keyword = remove_polish(word)
        openable = 1
        response = urlopen(uri)
        forms = ParseResponse(response, backwards_compat=False)
        if len(forms)==0:
            os.system("python PyCrawler.py"+" baza.db '"+ uri+"' 1 "+ keyword)
            return
        form = forms[0]

        # search for text input in form and put keyword there
        control = form.find_control(type="text")

        control.value = keyword

        # form.click() returns a mechanize.Request object
        # (see HTMLForm.click.__doc__ if you want to use only the forms support, and
        # not the rest of mechanize)
        request2 = form.click()  # mechanize.Request object
        try:
            response2 = urlopen(request2)
        except:
            print "Nie mozna otworzyc formularza"
            openable = 0
            pass

        #get the url of page
        if not openable:
            search_url=uri
        else:
            search_url = response2.geturl()

        #start crawler on it
        os.system("python PyCrawler.py"+" baza.db '"+ search_url+"' 1 "+ keyword)

def _get_results(form, dbg = False):
    # click the form
    clicked_form = form.click()
    # then get the results page
    result = mechanize.urlopen(clicked_form)

    #### EXPORTING RESULTS FILE
    # so what I do is that I fetch the first results page,
    # click the form/link to get all hits as a colon separated
    # ascii table file
    
    # get the form
    resultform = mechanize.ParseResponse(result, backwards_compat=False)
    result.close()
    resultform = resultform[0]
    # set colon as dilimeter of the table (could use anything I guess)
    #~ resultform.find_control('export_delimiter').items[1].selected =  True
    resultform.find_control('export_delimiter').toggle('colon')
    resultform_clicked = resultform.click()
    result_table = mechanize.urlopen(resultform_clicked)
    data = result_table.read()
    result_table.close()
    if dbg:
        return resultform, result_table, data
    else:
        return data

 def reply_to(self, postid, message):
     params = urllib.urlencode({
         'post_parent_id': postid,
         'snId': 0,
         'post_text': message
     })
     mechanize.urlopen(self.base_url + "/actions/post.php", params)

 def login(self, username, password):
     response = urlopen(urljoin(self.uri, "/?cs=login"))
     forms = ParseResponse(response, backwards_compat=False)
     form = forms[0]
     form.set_value(username, name='username')
     form.set_value(password, name='password')
     self.page = urlopen(form.click())

	def getDLurl(self, url):
		try:
			content = self.getUrl(url)
			match = re.findall('flashvars.playlist = \'(.*?)\';', content)
			if match:
				for url in match:
					url = 'http://ua.canna.to/canna/'+url
					content = self.getUrl(url)
					match = re.findall('<location>(.*?)</location>', content)
					if match:
						for url in match:
							url = 'http://ua.canna.to/canna/'+url
							req = mechanize.Request('http://ua.canna.to/canna/single.php')
							response = mechanize.urlopen(req)
							req = mechanize.Request(url)
							req.add_header('User-Agent', ' Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.3) Gecko/2008092417 Firefox/3.0.3')
							response = mechanize.urlopen(req)
							response.close()
							code=response.info().getheader('Content-Location')
							url='http://ua.canna.to/canna/avzt/'+code
							print url
							return url

		except urllib2.HTTPError, error:
			printl(error,self,"E")
			message = self.session.open(MessageBox, ("Fehler: %s" % error), MessageBox.TYPE_INFO, timeout=3)
			return False

def get_vorlage(session_id, url):
    try:
        response = mechanize.urlopen(mechanize.Request(url))
        pprint.pprint(response)
    except URLError:
        return
    forms = mechanize.ParseResponse(response, backwards_compat=False)
    for form in forms:
        # All forms are iterated. Might not all be attachment-related.
        for control in form.controls:
            if control.name == 'DT':
                print control.name, control.value
                request2 = form.click()
                try:
                    response2 = mechanize.urlopen(request2)
                    form_url = response2.geturl()
                    if "getfile.asp" in form_url:
                        #print "ERFOLG:", response2.info()
                        pdf = response2.read()
                        md5 = hashlib.md5(pdf).hexdigest()
                        scraperwiki.sqlite.save(
                            unique_keys=['session_id', 'dt', 'md5', 'size'],
                            data={'session_id': session_id, 'dt': control.value, 'md5': md5, 'size': len(pdf)})
                        continue
                except mechanize.HTTPError, response2:
                    print "HTTP-FEHLER :("
                except URLError:
                    pass

def loginRedmine():
	forms = ParseResponse(urlopen(URL_LOGIN))
	form = forms[0]
	form['username'] = USER_ID
	form['password'] = USER_PW
	request = form.click()
	mechanize.urlopen(request)

def connect(url, username, password):
    try:
        if connected(url):
            raise LinkException('You are already connected')
        try:
            response = urlopen(url)
        except URLError:
            raise SSIDException('You are not connected on a FON box')

        forms = ParseResponse(response, backwards_compat=False)

        try:
            form = forms[0]
            form["login[user]"] = username
            form["login[pass]"] = password
        except IndexError:
            raise SSIDException('You are not connected on a FON box')

        try:
            response_page = urlopen(form.click()).read()
        except NameError:
            raise SSIDException('You are not connected on a FON box')

        return not 'class="form_error"' in response_page
    except PlainURLError:
        if connected(url):
            return True
        else:
            raise RuntimeError("Connection failed.")

    def __get_csv(self, letter='a', now=False):

        #open the url
        current_url = self.overview_url + '1111&b=' + letter
        overview_req = mechanize.Request(current_url)
        overview_res = mechanize.urlopen(overview_req)

        #find the list of entries to post
        py_query = PyQuery(overview_res.read())
        titlelist = py_query("input[name='titelnrliste']").val()

        #create the post request
        post_data = {
            'url': current_url,
            'download': '[Download]',
            'titelnrliste': titlelist
        }

        if (now):
            #find the checked box (the current quartal)
            default_quartal = py_query(".quartal input:checked").attr('name')
            post_data[str(default_quartal)] = 'ON'
        else:
            #enable all quartal's checkbox
            quartals = [1, 2, 3, 4]
            for i in quartals:
                if i in range(1, 5):
                    post_data[str(self.year) + str(i)] = 'ON'

        #send the post request
        csv_req = mechanize.Request(current_url, urllib.urlencode(post_data))
        csv_res = mechanize.urlopen(csv_req)
        self.csv_parser.process_result(response=csv_res)

def rtnHTMLformat(tmpddGenrcgenPresent, sppPrefx, pthwcod, ouPthwpng):
    inpx = '\n'.join(tmpddGenrcgenPresent)  # inpx="ALDH2 color \nALDH3A1	color"
    request = mechanize.Request(
        "http://www.genome.jp/kegg/tool/map_pathway2.html")
    response = mechanize.urlopen(request)
    forms = mechanize.ParseResponse(response, backwards_compat=False)
    form = forms[0]
    form["unclassified"] = inpx
    form["org"] = sppPrefx
    request2 = form.click()
    response2 = mechanize.urlopen(request2)
    a = str(response2.read()).split('href="/kegg-bin/show_pathway?')[1]
    code = a.split('/')[0]  # response2.read()
    request = mechanize.Request(
        "http://www.genome.jp/kegg-bin/show_pathway?%s/%s.args" % (code, pthwcod))  # request=mechanize.Request("http://www.genome.jp/kegg-bin/show_pathway?%s/%s.args"%('13171478854246','hsa00410'))
    response = mechanize.urlopen(request)
    forms = mechanize.ParseResponse(response, backwards_compat=False)
    form = forms[1]
    status = ' NOT '
    try:
        imgf = str(forms[1]).split('/mark_pathway')[1].split('/')[0]
        os.system("wget --quiet http://www.genome.jp/tmp/mark_pathway%s/%s.png -O %s" % (imgf, pthwcod, ouPthwpng))
        status = ' '
    except:
        pass
    return 'A pathway image was%ssuccefully produced...' % status

def login(conf):
    try:
        username = conf.username
        password = conf.password
        request = mechanize.Request('%s/login.php' % conf.website)
        response = mechanize.urlopen(request, timeout=conf.timeout)
        forms = mechanize.ParseResponse(response)
        response.close()

        if len(forms) < 3:
            return Left('Failed to reach the login page.')

        form = forms[2]
        form['username'] = username
        form['password'] = password
        login_request = form.click()

        login_response = mechanize.urlopen(login_request, timeout=conf.timeout)
        logged_in = login_response.geturl() == ('%s/index.php'
                                                % conf.website)

        if not logged_in:
            return Left('Failed to log in with these credentials')

    except mechanize.HTTPError as resp:
        return Left('HTTPError when logging in: %s' % resp)
    except Exception as e:
        return Left('%s' % e)

    if conf.verbose: sys.stdout.write('Logged in as %s\n' % username)
    return Right('Logged in as %s' % username)

    def test_sending_headers(self):
        handler = self._make_request_handler([(200, [], "we don't care")])

        req = mechanize.Request("http://localhost:%s/" % handler.port,
                              headers={'Range': 'bytes=20-39'})
        mechanize.urlopen(req)
        self.assertEqual(handler.received_headers['Range'], 'bytes=20-39')

 def login1(self):
     self.brow = mechanize.Browser()
     
     httpHandler = mechanize.HTTPHandler()
     httpsHandler = mechanize.HTTPSHandler()
     
     httpHandler.set_http_debuglevel(DEBUG)
     self.cookiejar = mechanize.LWPCookieJar()
     #self.cookiejar = "Cookie    lzstat_uv=34741959842666604402|1786789; Hm_lvt_976797cb85805d626fc5642aa5244ba0=1304534271541; ASPSESSIONIDQCDRAQBB=JHCHINLAHGMAIGBIFMNANLGF; lzstat_ss=2189193215_2_1304564199_1786789; Hm_lpvt_976797cb85805d626fc5642aa5244ba0=1304535401191"
     self.opener = mechanize.OpenerFactory(mechanize.SeekableResponseOpener).build_opener(
                                     httpHandler,httpsHandler,
                                     mechanize.HTTPCookieProcessor(self.cookiejar),
                                     mechanize.HTTPRefererProcessor,
                                     mechanize.HTTPEquivProcessor,
                                     mechanize.HTTPRefreshProcessor,
                                     )
     self.opener.addheaders = [("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13"),
                          ("From", "")]
     #self.opener.addheaders = [(
      #                     "Referer", self.data['postUrl']
     #                      )]
     login={}
     login['method'] = self.data['method']
     login['name'] = self.data['name']
     login['pwd'] = self.data['pwd']
     loginUrl = self.data['loginUrl']+'?'+urllib.urlencode(login)
     print loginUrl
     response = mechanize.urlopen("http://esf.soufun.com/")
     response = mechanize.urlopen(loginUrl)
     print response.read().decode('gb2312')

def generate_script_sieve(request, group_id):
    group = get_object_or_404(Group, id=group_id)

    if request.method == 'POST':
        if group.always_data_id:
            # There is always_data mail id
            from mechanize import ParseResponse, urlopen, urljoin
            response = urlopen("https://admin.alwaysdata.com/login/")
            forms = ParseResponse(response, backwards_compat=False)
            login_form = forms[0]

            if settings.DEBUG:
                print login_form

            login_form["email"] = settings.ALWAYS_DATA_ID
            login_form["password"] = settings.ALWAYS_DATA_PASSWORD
            
            response = urlopen(login_form.click())
            url = 'https://admin.alwaysdata.com/email/%d/' % group.always_data_id
            response = urlopen(url)

            forms = ParseResponse(response, backwards_compat=False)

            if settings.DEBUG:
                for form in forms:
                    print form
            try:
                email_form = forms[1]
            except IndexError:
                messages.warning(request, _(u'%(group)s is not bind to alwaysdata yet (wrong password)' % 
                                          {'group': group}))
                
                return HttpResponseRedirect(reverse("group-detail", args=[group.pk]))

            email_form['sieve_filter'] = request.POST['filter_sieve'].encode('utf-8')

            req = email_form.click()
            req.add_header("Referer", url)
            response = urlopen(req)

            messages.success(request, _(u'Alwaysdata has been updated'))

        else:
            messages.warning(request, _(u'%(group)s is not bind to alwaysdata yet' % 
                                        {'group': group}))

        return HttpResponseRedirect(reverse("group-detail", args=[group.pk]))
            
    else:
        filter_sieve = export_sieve_configuration(group.contacts.all())
        
        context = get_global_context_data(Group, Group._meta.app_label)
        context['object_list'] = Group.objects.all()
        context['object'] = group
        context['filter_sieve'] = filter_sieve
        
        return render_to_response('contact/contact-sieve.html',
                                  context,
                                  context_instance=RequestContext(request))

    def test_404(self):
        expected_response = 'Bad bad bad...'
        handler = self._make_request_handler([(404, [], expected_response)])

        try:
            mechanize.urlopen('http://localhost:%s/weeble' % handler.port)
        except mechanize.URLError, f:
            pass

def get_nyc_legislation():  #search_terms=''
    book = xlwt.Workbook(encoding='utf-8', style_compression = 0)
    sheet = book.add_sheet('Legi', cell_overwrite_ok = True)  
    row=-1

    for items in ['smoking']:
        url = r'http://legistar.council.nyc.gov/Legislation.aspx'
        request = mechanize.Request(url)
        response = mechanize.urlopen(request)
        forms = mechanize.ParseResponse(response, backwards_compat=False)
        form = forms[0]
        response.close()

        form['ctl00$ContentPlaceHolder1$txtSearch'] = items
        submit_page = mechanize.urlopen(form.click())
        soup = BeautifulSoup(submit_page.read())

        for link in soup.find_all("a"):
            legislation = link.get("href")
            try:
                if 'LegislationDetail' in legislation:
                    url_stem = 'http://legistar.council.nyc.gov/'
                    url2 = "%s%s" % (url_stem, legislation)

                    request2 = requests.get(url2)
                    soup2 = BeautifulSoup(request2.content)
                    type = soup2.find_all("span",{"id":"ctl00_ContentPlaceHolder1_lblType2"})
                    status = soup2.find_all("span",{"id":"ctl00_ContentPlaceHolder1_lblStatus2"})
                    print url2

                    if ((type[0].text == "Resolution" or 
                        type[0].text == "Introduction") and 
                        (status[0].text == "Adopted")):

                        legislation_title = soup2.find_all("span",{"id":"ctl00_ContentPlaceHolder1_lblName2"})
                        legislation_date = soup2.find_all("span",{"id":"ctl00_ContentPlaceHolder1_lblOnAgenda2"})
                        legislation_committee = soup2.find_all("a",{"id":"ctl00_ContentPlaceHolder1_hypInControlOf2"})
                        legislation_text = soup2.find_all("span",{"class":"st1"})                                

                        legi_url, title, date, committee, text = ([] for i in range(5))
                        row = row + 1 

                        legi_url = url2                
                        for item in legislation_title:
                            title.append(item.text)
                        for item in legislation_date:
                            date.append(item.text)
                        for item in legislation_committee:
                            committee.append(item.text)
                        for item in legislation_text:
                            text.append(' '+item.text)

                        legi = [legi_url,title,date,committee,text]
                        for column, var_observ in enumerate(legi):
                            sheet.write (row, column, var_observ)
            except:
                pass
    book.save("legislation_data.xls")

def install_phpBB():
    print "installing phpBB..."
    #create forum
    url = "http://" + ip + "/phpBB3/"
    install_url = url + "install/index.php?mode=install&sub="
    br = mechanize.Browser()
    post = "dbms=mysqli&dbhost=" + ip  +  "&dbport=" + port +  "&dbname=cryptdb_phpbb&dbuser=root&dbpasswd=letmein&table_prefix=phpbb_&admin_name=admin&admin_pass1=letmein&admin_pass2=letmein&[email protected]&[email protected]"
    config = mechanize.urlopen(install_url+"config_file", data=post);
    br.set_response(config)
    post += "&email_enable=1&smtp_delivery=0&smtp_host=&smtp_auth=PLAIN&smtp_user=&smtp_pass=&cookie_secure=0&force_server_vars=0&server_protocol=http://&server_name=18.26.5.16&server_port=80&script_path=/phpBB"
    advanced = mechanize.urlopen(install_url+"advanced", data=post);
    br.set_response(advanced)
    br.select_form(nr=0)
    br.submit()
    br.select_form(nr=0)
    br.submit()

    os.system("mv $EDBDIR/../apps/phpBB3/install $EDBDIR/../apps/phpBB3/install2")

    print "logging in..."
    #login
    br.open(url+"ucp.php?mode=login")
    br.select_form(nr=1)
    br["username"] = "admin"
    br["password"] = "letmein"
    br.submit()
    print "to ACP..."
    #authenticate to go to ACP
    br.follow_link(text="Administration Control Panel")
    br.select_form(nr=1)
    i = str(br.form).find("password")
    j = str(br.form).find("=)",i)
    br[str(br.form)[i:j]] = "letmein"
    br.submit()
    print "getting permissions page..."
    #navigate to group permissions
    br.follow_link(text="Permissions")
    br.follow_link(text="Groups\xe2\x80\x99 permissions")
    #select Newly Registered Users
    br.select_form(nr=0)
    br["group_id[]"] = ["7"]
    br.submit()
    #set all permissions to yes
    print "setting permissions..."
    br.select_form(nr=1)
    i = 1
    while i > 0:
        start = str(br.form).find("setting[7][0][",i)
        if (start < 0):
            break
        end = str(br.form).find("=[",start)
        if (end < 0):
            break
        br[str(br.form)[start:end]] = ["1"]
        i = end
    br.submit()

def doLogin():
    url = "https://awesome-hd.net/login.php"
    response = mechanize.urlopen(url)

    forms = mechanize.ParseResponse(response, backwards_compat=False)
    form = forms[0]
    form["username"] = s.username
    form["password"] = s.password

    mechanize.urlopen(form.click())

    def isThereInternet(self):
        try:
            mechanize.urlopen('http://github.com', timeout=1)
            return True
        except mechanize.URLError as e:
            print "There is no internet {}".format(e)

            self.error_msg.setText("You are not connected to the internet")
            self.error_msg.setDetailedText("This feature will not work without an internet connection. ")
            self.error_msg.exec_()
            return False