def addMissingCommonsWikidataLink(self, filename, wikidataitem):
        """
        Try to add a missing link to Commons. Returns True if it worked and False if it failed
        """
        filepage = pywikibot.FilePage(self.commons, title=filename)

        text = filepage.get()
        replaceregex = u'\{\{(Artwork|Painting|Art Photo|Google Art Project|Google Cultural Institute|Walters Art Museum artwork|NARA-image-full)'
        emptywikidataregex = u'(\s*\|\s*wikidata\s*=)\s*\n'
        wikidataregex = u'[wW]ikidata\s*=\s*(Q\d+)\s*'

        pywikibot.output(u'Working on %s' % (filepage.title(),))

        wdmatch = re.search(wikidataregex, text)

        if wdmatch:
            # Template with duplicate template problems might hit this one or when database query is a bit stale
            pywikibot.output(u'Seems to already link to Wikidata %s' % (wdmatch.group(1),))
            return False

        # First try to update an existing field
        newtext = re.sub(emptywikidataregex, u'\\1%s\n' % (wikidataitem,), text, count=1)

        if text==newtext:
            #Ok, that didn't work, just slap it at the top
            newtext = re.sub(replaceregex, u'{{\\1\n|wikidata=%s' % (wikidataitem,), text, count=1, flags=re.I)
            if text==newtext:
                pywikibot.output(u'Unable to add Wikidata link to %s' % (filename,))
                return False

        pywikibot.showDiff(text, newtext)
        summary = u'Adding link to [[:d:%s]] based on usage on that item' % (wikidataitem,)
        pywikibot.output(summary)
        filepage.put(newtext, summary=summary)
        return True

    def add_template(self):
        if not self.adtTitle:
            return  # silently fail

        adtPage = pywikibot.Page(self.site, self.adtTitle, ns=1)
        code = mwparserfromhell.parse(adtPage.text)

        war_adt_added = False
        for template in code.filter_templates(recursive=False):
            if template.name.matches("AdT-Vorschlag Hinweis"):
                code.remove(template)
                pywikibot.output(u'D:AdT: {{AdT-Vorschlag Hinweis}} gefunden,'
                                 u'entfernt')
            if template.name.matches("War AdT"):
                if not any(self.snapDate in p for p in template.params):
                    template.add(str(len(template.params)+1), self.snapDate)
                    pywikibot.output(u'D:AdT: {{War AdT}} '
                                     u'gefunden, füge heute hinzu')
                war_adt_added = True
        text = unicode(code)
        if not war_adt_added:
            template = u'{{War AdT|1=' + self.snapDate + u'}}\n'
            text = self.__add_templ(text, template)

        if adtPage.text != text:
            pywikibot.showDiff(adtPage.text, text)  # debug
            adtPage.text = text
            if not self.dry:
                adtPage.save(comment=templateComment, botflag=True, minor=True)

 def process_page(self, page):
     talk_page = page
     page = talk_page.toggleTalkPage()
     #find the edit where {{good article}] was added
     found_oldid = False
     oldid = None
     while not found_oldid:
         self.site.loadrevisions(page, getText=True, rvdir=False,
                                 step=10, total=10, startid=oldid)
         hist = page.fullVersionHistory(total=10)  # This should fetch nothing...
         for revision in hist:
             if re.search('\{\{(good|ga) article\}\}', revision[3], re.IGNORECASE):
                 oldid = revision[0]
             else:
                 #current oldid is the right one
                 found_oldid = True
                 break
     #add the oldid in the template
     if not oldid:
         self.output('* ERROR: Could not find oldid for [[%s]]' % talk_page.title())
         return
     self.output('* Adding |oldid=%s to [[%s]]' % (oldid, talk_page.title()))
     oldtext = talk_page.get()
     search = re.search('\{\{GA\s?\|(.*?)\}\}', oldtext)
     newtext = oldtext.replace(search.group(0), '{{GA|%s|oldid=%s}}' % (search.group(1), oldid))
     pywikibot.showDiff(oldtext, newtext)
     talk_page.put(newtext, 'BOT: Adding |oldid=%s to {{[[Template:GA|GA]]}}' % oldid)

    def tagNowcommons(self, imagepage, filename):
        """Tagged the imag which has been moved to Commons for deletion."""
        if pywikibot.Page(pywikibot.Site('commons', 'commons'),
                          u'File:' + filename).exists():
            # Get a fresh copy, force to get the page so we dont run into edit
            # conflicts
            imtxt = imagepage.get(force=True)

            # Remove the move to commons templates
            if imagepage.site.language() in moveToCommonsTemplate:
                for moveTemplate in moveToCommonsTemplate[
                        imagepage.site.language()]:
                    imtxt = re.sub(u'(?i)\{\{' + moveTemplate +
                                   u'[^\}]*\}\}', u'', imtxt)

            # add {{NowCommons}}
            if imagepage.site.language() in nowCommonsTemplate:
                addTemplate = nowCommonsTemplate[
                    imagepage.site.language()] % filename
            else:
                addTemplate = nowCommonsTemplate['_default'] % filename

            commentText = i18n.twtranslate(
                imagepage.site(), 'commons-file-now-available',
                {'localfile': imagepage.title(withNamespace=False),
                 'commonsfile': filename})

            pywikibot.showDiff(imagepage.get(), imtxt + addTemplate)
            imagepage.put(imtxt + addTemplate, comment=commentText)

    def userPut(self, page, oldtext, newtext):
        """
        Print differences, ask user for confirmation,
        and puts the page if needed.

        Option used:
            * 'always'
        """
        if oldtext == newtext:
            pywikibot.output(u'No changes were needed on %s'
                             % page.title(asLink=True))
            return

        pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
                         % page.title())
        pywikibot.showDiff(oldtext, newtext)

        choice = 'a'
        if not self.getOption('always'):
            choice = pywikibot.inputChoice(
                u'Do you want to accept these changes?',
                ['Yes', 'No', 'All'],
                ['y', 'N', 'a'],
                'N'
            )
            if choice == 'a':
                # Remember the choice
                self.options['always'] = True

        if choice != 'n':
            page.put(newtext, async=(choice == 'a'))

    def save(self, page, text):
        if text != page.get():
            # Show the title of the page we're working on.
            # Highlight the title in purple.
            pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
                             % page.title())
            pywikibot.showDiff(page.get(), text)
            if not self.always:
                choice = pywikibot.inputChoice(
                    u'Do you want to accept these changes?',
                    ['Yes', 'No', 'Always yes'], ['y', 'N', 'a'], 'N')
                if choice == 'n':
                    return
                elif choice == 'a':
                    self.always = True

            if self.always:
                try:
                    page.put(text, comment=self.comment)
                except pywikibot.EditConflict:
                    pywikibot.output(u'Skipping %s because of edit conflict'
                                     % (page.title(),))
                except pywikibot.SpamfilterError, e:
                    pywikibot.output(
                        u'Cannot change %s because of blacklist entry %s'
                        % (page.title(), e.url))
                except pywikibot.LockedPage:
                    pywikibot.output(u'Skipping %s (locked page)'
                                     % (page.title(),))

 def save(self, text, page, comment, minorEdit=True, botflag=True):
     """Save the text."""
     if text != page.text:
         # Show the title of the page we're working on.
         # Highlight the title in purple.
         pywikibot.output(color_format(
             '\n\n>>> {lightpurple}{0}{default} <<<', page.title()))
         # show what was changed
         pywikibot.showDiff(page.get(), text)
         pywikibot.output(u'Comment: %s' % comment)
         if not self.dry:
             if pywikibot.input_yn(
                     u'Do you want to accept these changes?',
                     default=False, automatic_quit=False):
                 page.text = text
                 try:
                     # Save the page
                     page.save(summary=comment, minorEdit=minorEdit,
                               botflag=botflag)
                 except pywikibot.LockedPage:
                     pywikibot.output(u"Page %s is locked; skipping."
                                      % page.title(asLink=True))
                 except pywikibot.EditConflict:
                     pywikibot.output(
                         u'Skipping %s because of edit conflict'
                         % (page.title()))
                 except pywikibot.SpamfilterError as error:
                     pywikibot.output(
                         u'Cannot change %s because of spam blacklist entry '
                         u'%s' % (page.title(), error.url))
                 else:
                     return True

def creditByFlickrUrl():
    commons = pywikibot.Site('commons', 'commons')
    flickrurls = [
        'http://flickr.com/people/[email protected]',
        'http://www.flickr.com/people/[email protected]',
        'https://flickr.com/people/[email protected]',
        'https://www.flickr.com/people/[email protected]',
    ]
    for flickrurl in flickrurls:
        images = ['hola']
        while images:
            linksearch = 'https://commons.wikimedia.org/w/index.php?target=%s&title=Special:LinkSearch' % (flickrurl)
            req = urllib.request.Request(linksearch, headers={ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0' })
            raw = urllib.request.urlopen(req).read().strip().decode('utf-8')
            images = re.findall(r'title="(File:[^<>]+?)">File:', raw)
            print(images)
            for image in images:
                page = pywikibot.Page(commons, image)
                text = page.text
                newtext = page.text
                newtext = re.sub(r'(\|\s*Author\s*=\s*)\[%s [^\]]*?\]\s*(de|from)?\s*(España|Spain)?' % (flickrurl), r'\1{{User:Emijrp/credit}}', newtext)
                if text != newtext:
                    pywikibot.showDiff(text, newtext)
                    page.text = newtext
                    page.save('BOT - Updating credit template')

def appendtext(page, apptext, always):
    if page.isRedirectPage():
        page = page.getRedirectTarget()
    if not page.exists():
        if page.isTalkPage():
            text = u''
        else:
            raise pywikibot.NoPage(u"Page '%s' does not exist" % page.title())
    else:
        text = page.text
    # Here you can go editing. If you find you do not
    # want to edit this page, just return
    oldtext = text
    text += apptext
    if text != oldtext:
        pywikibot.showDiff(oldtext, text)
        if not always:
            choice = pywikibot.inputChoice(
                u'Do you want to accept these changes?', ['Yes', 'No', 'All'],
                'yNa', 'N')
            if choice == 'a':
                always = True
        if always or choice == 'y':
            page.text = text
            page.save(i18n.translate(pywikibot.Site(), comment,
                                     fallback=True))

    def cleanup_templates(self):
        for adt in self.erl_props:
            if adt in self.props:
                # mehrmals für AdT vorgeschlagen
                continue

            page = pywikibot.Page(self.site, adt, ns=1)

            if not page.exists():
                pywikibot.error(u'ERROR: disc for AdT-Vorschlag ' + adt
                                + u' does not exist!')
                return

            oldtext = page.text
            code = mwparser.parse(page.text)

            for template in code.filter_templates(recursive=False):
                if template.name.matches("AdT-Vorschlag Hinweis"):
                    code.remove(template)
                    pywikibot.output(adt +
                                     u': {{AdT-Vorschlag Hinweis}} '
                                     u'gefunden, entfernt')
            page.text = unicode(code)
            if page.text == oldtext:
                continue

            page.text = page.text.lstrip(u'\n')
            pywikibot.showDiff(oldtext, page.text)
            comment = u'Bot: [[Vorlage:AdT-Vorschlag Hinweis]] entfernt'
            if not self.dry:
                page.save(comment=comment, botflag=True, minor=True)

 def treat(self, page):
     # Show the title of the page we're working on.
     # Highlight the title in purple.
     pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
                      % page.title())
     try:
         oldText = page.get()
         text = oldText
         curpos = 0
         while curpos < len(text):
             match = self.linkR.search(text, pos=curpos)
             if not match:
                 break
             # Make sure that next time around we will not find this same
             # hit.
             curpos = match.start() + 1
             text, jumpToBeginning = self.handleNextLink(text, match)
             if jumpToBeginning:
                 curpos = 0
         if oldText == text:
             pywikibot.output(u'No changes necessary.')
         else:
             pywikibot.showDiff(oldText, text)
             page.text = text
             page.save(self.comment)
     except pywikibot.NoPage:
         pywikibot.output(u"Page %s does not exist?!"
                          % page.title(asLink=True))
     except pywikibot.IsRedirectPage:
         pywikibot.output(u"Page %s is a redirect; skipping."
                          % page.title(asLink=True))
     except pywikibot.LockedPage:
         pywikibot.output(u"Page %s is locked?!" % page.title(asLink=True))

 def run(self):
     for template in self._templates_generator():
         title = template.title(withNamespace=False).lower()
         for page in pagegenerators.ReferringPageGenerator(template, onlyTemplateInclusion=True):
             try:
                 text = page.get()
             except pywikibot.Error:
                 continue
             else:
                 code = mwparserfromhell.parse(text)
             for temp in code.ifilter_templates():
                 if temp.has_param("nosubst") or temp.has_param("demo"):
                     continue
                 replace = False
                 if temp.name.lower().strip() == title:
                     replace = True
                 if temp.name.lower().strip().startswith("template:") and temp.name.lower().strip()[9:] == title:
                     replace = True
                 if replace:
                     temp.name = "subst:%s" % template.title()
                     temp.add("subst", "subst:")
             if text != code:
                 pywikibot.showDiff(text, code)
                 try:
                     page.put(
                         code,
                         "Bot: Substituting {{%s}}" % template.title(asLink=True, allowInterwiki=False)
                     )
                 except pywikibot.Error:
                     continue

    def save(self, page, newText):
        """
        Saves the page to the wiki, if the user accepts the changes made.
        """
        pywikibot.showDiff(page.get(), newText)
        if not self.always:
            choice = pywikibot.inputChoice(
                u'Do you want to accept these changes?',
                ['Yes', 'No', 'Always yes'], ['y', 'N', 'a'], 'Y')
            if choice == 'n':
                return
            elif choice == 'a':
                self.always = True

        if self.always:
            try:
                page.put(newText)
            except pywikibot.EditConflict:
                pywikibot.output(u'Skipping %s because of edit conflict'
                                 % (page.title(),))
            except pywikibot.SpamfilterError, e:
                pywikibot.output(
                    u'Cannot change %s because of blacklist entry %s'
                    % (page.title(), e.url))
            except pywikibot.LockedPage:
                pywikibot.output(u'Skipping %s (locked page)' % (page.title(),))

def replaceSubpageWithDimensions(sourcePrefix, destPrefix):
	generator = pagegenerators.PrefixingPageGenerator(prefix=sourcePrefix)
	for page in generator:
		title = page.title()
		suffix = title[title.find('/') + 1:]
		repl = u"{{" + title
		repl2 = u"{{" + title.replace(u"Format:", u"")
		repl3 = u"{{" + title.replace(u"Format:", u"").replace(u"'", u"'")
		repl4 = u"{{" + title.replace(u"Format:", u"").lower().replace(suffix.lower(), suffix)
		try:
			tpl = sf.tl2Dict(sf.extractTemplate(page.get(), u"Demografia"))[0]
			x = tpl["dimx"].strip()
			y = tpl["dimy"].strip()
		except:
			continue
		call = u"{{" + destPrefix + u'|' + suffix + u'|lățime=' + x + u'|înălțime=' + y
		generator2 = pagegenerators.ReferringPageGenerator(page)
		for page2 in generator2:
			if page2.namespace() > 0:
				continue
			try:
				text = page2.get()
				newtext = text.replace(repl, call)
				newtext = newtext.replace(repl2, call)
				newtext = newtext.replace(repl3, call)
				newtext = newtext.replace(repl4, call)
				if newtext != text:
					pywikibot.showDiff(text, newtext)
					page2.put(newtext, comment="Folosesc noul format pentru grafice de demografie")
			except:
				continue

def doTagging( imageList, startTag, endTag, summary ) :
  global site, debug
  for image in imageList :
    page = pywikibot.Page(site, image )

    if page.exists() :
      print "Tagging " + image.encode("utf-8")

      # follow redirects to the actual image page
      while True :
        try :
          text = page.get()
          break
        except pywikibot.IsRedirectPage :
          page = page.getRedirectTarget()

      # already tagged maybe?
      oldtext = text.replace(unassessedCat, '')
      if string.find(text, startTag ) < 0 :
        text += "\n" + startTag + endTag + "\n"
        if not debug:
          #try :
          #  page.put(text, comment=summary, minorEdit=False)
          #except pywikibot.LockedPage :
          #  print image.encode("utf-8") + " has an editprotected description page!"
          tryPut(page,text,summary)
        else:
          pywikibot.output(u">>> \03{lightpurple}%s\03{default} <<<" % page.title())
          pywikibot.showDiff(oldtext, text)
    else :
      print "Oops " + image.encode("utf-8") + " doesn't exist..."

 def treat(self, page):
     """Remove links pointing to the configured page from the given page."""
     self.current_page = page
     try:
         oldText = page.get()
         text = oldText
         curpos = 0
         while curpos < len(text):
             match = self.linkR.search(text, pos=curpos)
             if not match:
                 break
             # Make sure that next time around we will not find this same
             # hit.
             curpos = match.start() + 1
             text, jumpToBeginning = self.handleNextLink(text, match)
             if jumpToBeginning:
                 curpos = 0
         if oldText == text:
             pywikibot.output(u'No changes necessary.')
         else:
             pywikibot.showDiff(oldText, text)
             page.text = text
             page.save(self.comment)
     except pywikibot.NoPage:
         pywikibot.output(u"Page %s does not exist?!"
                          % page.title(asLink=True))
     except pywikibot.IsRedirectPage:
         pywikibot.output(u"Page %s is a redirect; skipping."
                          % page.title(asLink=True))
     except pywikibot.LockedPage:
         pywikibot.output(u"Page %s is locked?!" % page.title(asLink=True))

def crawl():
    a = Canagram()
    for page in file(batchfile,'r').readlines():
        try:
            page = page.replace('\n','')
            wikipedia.output('>>> %s <<<'%page)
            if page:
                if a.getAnagramsFor(page.title()):
                    page = wikipedia.Page('mg', page.decode('utf8'))
                    i = page.get()
                else:
                    continue
            else:
                print "nahitana hadisoana : anaram-pejy tsy azo ampiasaina."
                continue

            if i:
                c=insert(i, page.title())
            else: continue
            if c:
                wikipedia.output('>>> %s <<<'%page.title())
                wikipedia.showDiff(i, c)
                if i!=c:
                    page.put(c, '+anagrama')
                    pass
                else:
                    print "tsy hisy fiovana"
            else:
                print "tsy hisy fiovana" 
        except SyntaxError:
            print "Nisy hadisoana..."
            continue

    def userPut(self, page, oldtext, newtext, **kwargs):
        """
        Save a new revision of a page, with user confirmation as required.

        Print differences, ask user for confirmation,
        and puts the page if needed.

        Option used:
            * 'always'
        """
        if oldtext == newtext:
            pywikibot.output(u'No changes were needed on %s'
                             % page.title(asLink=True))
            return

        self.current_page = page
        pywikibot.showDiff(oldtext, newtext)
        if 'comment' in kwargs:
            pywikibot.output(u'Comment: %s' % kwargs['comment'])

        if not self.user_confirm('Do you want to accept these changes?'):
            return

        if 'async' not in kwargs and self.getOption('always'):
            kwargs['async'] = True

        page.text = newtext
        page.save(**kwargs)

 def update(self, push=True):
     self.fetch_info()
     self.parse_info()
     print self.LOCATION
     print self.CATEGORY
     print self.ABOUT
     print self.MOVEMENT
     print self.PRESSURE
     print self.WINDS
     #print self.UTC_TIMESTAMP
     #actually update crap
     #return
     text = self.wikipage.get()
     code = mwparserfromhell.parse(text)
     main = pywikibot.Page(self.wikipage.site, '2012 Atlantic hurricane season')
     main_text = main.get()
     main_code = mwparserfromhell.parse(main_text)
     for template in code.filter_templates():
         name = template.name.lower().strip()
         if name == 'Infobox hurricane current'.lower():
             if template.get('name').value.strip() == 'Hurricane Sandy':
                 template.get('time').value = self.UTC_TIMESTAMP
                 template.get('category').value = self.CATEGORY
                 template.get('gusts').value = self.format_wind(self.WINDS)
                 template.get('lat').value = self.LOCATION['latc']
                 template.get(1).value = self.LOCATION['latd']
                 template.get('lon').value = self.LOCATION['lonc']
                 template.get(2).value = self.LOCATION['lond']
                 template.get('movement').value = self.format_movement(self.MOVEMENT)
                 template.get('pressure').value = self.format_pressure(self.PRESSURE)
     pywikibot.showDiff(text, unicode(code))
     if push:
         self.wikipage.put(unicode(code), 'Bot: Updating hurricane infobox. Errors? [[User talk:Legoktm|report them!]]')

 def save(self, text, page, comment=None, minorEdit=True,
          botflag=True):
     """Update the given page with new text."""
     # only save if something was changed
     if text != page.get():
         # Show the title of the page we're working on.
         # Highlight the title in purple.
         pywikibot.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
                          % page.title())
         # show what was changed
         pywikibot.showDiff(page.get(), text)
         pywikibot.output(u'Comment: %s' % comment)
         if not self.dry:
             try:
                 page.text = text
                 # Save the page
                 page.save(summary=comment or self.comment,
                           minor=minorEdit, botflag=botflag)
             except pywikibot.LockedPage:
                 pywikibot.output(u"Page %s is locked; skipping."
                                  % page.title(asLink=True))
             except pywikibot.EditConflict:
                 pywikibot.output(
                     u'Skipping %s because of edit conflict'
                     % (page.title()))
             except pywikibot.SpamfilterError as error:
                 pywikibot.output(
                     u'Cannot change %s because of spam blacklist entry %s'
                     % (page.title(), error.url))
             else:
                 return True
     return False