def getPhotos(photoset=u'', start_id='', end_id='', interval=100):
    """Loop over a set of Panoramio photos."""
    i = 0
    has_more = True
    url = ('http://www.panoramio.com/map/get_panoramas.php?'
           'set=%s&from=%s&to=%s&size=original')
    while has_more:
        gotInfo = False
        maxtries = 10
        tries = 0
        while not gotInfo:
            try:
                if tries < maxtries:
                    tries += 1
                    panoramioApiPage = urlopen(url % (photoset, i, i + interval))
                    contents = panoramioApiPage.read().decode('utf-8')
                    gotInfo = True
                    i += interval
                else:
                    break
            except IOError:
                pywikibot.output(u'Got an IOError, let\'s try again')
            except socket.timeout:
                pywikibot.output(u'Got a timeout, let\'s try again')

        metadata = json.loads(contents)
        photos = metadata.get(u'photos')
        for photo in photos:
            yield photo
        has_more = metadata.get(u'has_more')
    return

    def _do_insert(self, valuesdict):
        sqlreq = u"insert into `%(DB)s`.`%(table)s` ("%self.infos
        for i in valuesdict:
            sqlreq += u"`%s`,"%self.connect.escape_string(i)
        sqlreq = sqlreq.strip(',')
        sqlreq += u") values ("
        for i in valuesdict:
            valuesdict[i]=valuesdict[i].replace("'","\\'")
            sqlreq += u"'%s',"%valuesdict[i]
        sqlreq = sqlreq.strip(',')
        sqlreq += u")"
        try:
            self.cursor.execute(sqlreq)
        except UnicodeError:
            sqlreq = sqlreq.encode('utf8')
            self.cursor.execute(sqlreq)
        except Exception as e:
            if verbose: wikipedia.output(sqlreq)
            raise e

        self.querycount+=1
        if not self.querycount%1000:
            qcstr = str(self.querycount)
            qcstr = qcstr + chr(8)*(len(qcstr)+1)
            if verbose: print qcstr,

def isUncat(page):
    """
    Do we want to skip this page.

    If we found a category which is not in the ignore list it means
    that the page is categorized so skip the page.
    If we found a template which is in the ignore list, skip the page.
    """
    pywikibot.output(u'Working on ' + page.title())

    for category in page.categories():
        if category not in ignoreCategories:
            pywikibot.output(u'Got category ' + category.title())
            return False

    for templateWithTrail in page.templates():
        # Strip of trailing garbage
        template = templateWithTrail.title().rstrip('\n').rstrip()
        if template in skipTemplates:
            # Already tagged with a template, skip it
            pywikibot.output(u'Already tagged, skip it')
            return False
        elif template in ignoreTemplates:
            # template not relevant for categorization
            pywikibot.output(u'Ignore ' + template)
        else:
            pywikibot.output(u'Not ignoring ' + template)
            return False
    return True

    def run(self):
        """
        Starts the robot.
        """

        for imagePage in self.generator:
            pywikibot.output(u'Working on %s' % (imagePage.title(),))
            if imagePage.title(withNamespace=False) in self.withImage:
                pywikibot.output(u'Image is already in use in item %s' % (self.withImage.get(imagePage.title(withNamespace=False),)))
                continue

            text = imagePage.get()
            regex = '\s*\|\s*accession number\s*=\s*([^\s]+)\s*'
            match = re.search(regex, text)
            if match:
                paintingId = match.group(1).strip()
                pywikibot.output(u'Found ID %s on the image' % (paintingId,))

                if paintingId in self.withoutImage:
                    pywikibot.output(u'Found an item to add it to!')

                    paintingItemTitle = u'Q%s' % (self.withoutImage.get(paintingId),)
                    paintingItem = pywikibot.ItemPage(self.repo, title=paintingItemTitle)
                    paintingItem.get()

                    if u'P18' not in paintingItem.claims:
                        newclaim = pywikibot.Claim(self.repo, u'P18')
                        newclaim.setTarget(imagePage)
                        pywikibot.output('Adding image claim to %s' % paintingItem)
                        summary = u'Adding image based on %s' % (paintingId,)
                        paintingItem.addClaim(newclaim, summary=summary)

    def test_archivebot(self, code=None):
        """Test archivebot for one site."""
        site = self.get_site(code)
        if code != 'de':  # bug T69663
            page = pywikibot.Page(site, 'user talk:xqt')
        else:
            page = pywikibot.Page(site, 'user talk:ladsgroup')
        talk = archivebot.DiscussionPage(page, None)
        self.assertIsInstance(talk.archives, dict)
        self.assertIsInstance(talk.archived_threads, int)
        self.assertTrue(talk.archiver is None)
        self.assertIsInstance(talk.header, basestring)
        self.assertIsInstance(talk.timestripper, TimeStripper)

        self.assertIsInstance(talk.threads, list)
        self.assertGreaterEqual(
            len(talk.threads), THREADS[code],
            u'{0:d} Threads found on {1!s},\n{2:d} or more expected'.format(len(talk.threads), talk, THREADS[code]))

        for thread in talk.threads:
            self.assertIsInstance(thread, archivebot.DiscussionThread)
            self.assertIsInstance(thread.title, basestring)
            self.assertIsInstance(thread.now, datetime)
            self.assertEqual(thread.now, talk.now)
            self.assertIsInstance(thread.ts, TimeStripper)
            self.assertEqual(thread.ts, talk.timestripper)
            self.assertIsInstance(thread.code, basestring)
            self.assertEqual(thread.code, talk.timestripper.site.code)
            self.assertIsInstance(thread.content, basestring)
            try:
                self.assertIsInstance(thread.timestamp, datetime)
            except AssertionError:
                if thread.code not in self.expected_failures:
                    pywikibot.output('code {0!s}: {1!s}'.format(thread.code, thread.content))
                raise

def main(*args):
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: list of unicode
    """
    all = False
    new = False
    sysop = False
    for arg in pywikibot.handle_args(args):
        if arg in ('-all', '-update'):
            all = True
        elif arg == '-new':
            new = True
        elif arg == '-sysop':
            sysop = True
    if all:
        refresh_all(sysop=sysop)
    elif new:
        refresh_new(sysop=sysop)
    else:
        site = pywikibot.Site()
        watchlist = refresh(site, sysop=sysop)
        pywikibot.output(u'{0:d} pages in the watchlist.'.format(len(watchlist)))
        for page in watchlist:
            try:
                pywikibot.stdout(page.title())
            except pywikibot.InvalidTitle:
                pywikibot.exception()

    def addReleased(self, item, imdbid):
        '''
        Add the first airdate to the item based on the imdbid
        '''
        pywikibot.output(u'Trying to add date to %s based on %s' % (item, imdbid))
        data = item.get()
        claims = data.get('claims')
        if u'P1191' in claims:
            return True
        if imdbid not in self.imdbcache:
            return False
        releasedate = self.imdbcache[imdbid].get('released')
        regex = u'^(\d\d\d\d)-(\d\d)-(\d\d)$'
        match = re.match(regex, releasedate)
        if not match:
            return False

        newdate = pywikibot.WbTime(year=int(match.group(1)),
                                    month=int(match.group(2)),
                                    day=int(match.group(3)),)

        newclaim = pywikibot.Claim(self.repo, u'P1191')
        newclaim.setTarget(newdate)
        pywikibot.output('Adding release date claim %s to %s' % (releasedate, item))
        item.addClaim(newclaim)
        refurl = pywikibot.Claim(self.repo, u'P854')
        refurl.setTarget(u'http://www.omdbapi.com/?i=%s' % (imdbid,))
        refdate = pywikibot.Claim(self.repo, u'P813')
        today = datetime.datetime.today()
        date = pywikibot.WbTime(year=today.year, month=today.month, day=today.day)
        refdate.setTarget(date)
        newclaim.addSources([refurl, refdate])

    def findCommonscatLink(self, page=None):
        """Find CommonsCat template on interwiki pages.

        In Pywikibot 2.0, page.interwiki() now returns Link objects,
        not Page objects

        @rtype: unicode, name of a valid commons category
        """
        for ipageLink in page.langlinks():
            ipage = pywikibot.page.Page(ipageLink)
            pywikibot.log("Looking for template on %s" % (ipage.title()))
            try:
                if (not ipage.exists() or ipage.isRedirectPage() or
                        ipage.isDisambig()):
                    continue
                commonscatLink = self.getCommonscatLink(ipage)
                if not commonscatLink:
                    continue
                (currentTemplate,
                 possibleCommonscat, linkText, Note) = commonscatLink
                checkedCommonscat = self.checkCommonscatLink(possibleCommonscat)
                if (checkedCommonscat != u''):
                    pywikibot.output(
                        u"Found link for %s at [[%s:%s]] to %s."
                        % (page.title(), ipage.site.code,
                           ipage.title(), checkedCommonscat))
                    return checkedCommonscat
            except pywikibot.BadTitle:
                # The interwiki was incorrect
                return u''
        return u''

 def run(self):
     # If the enable page is set to disable, turn off the bot
     # (useful when the bot is run on a server)
     if not self.enable_page():
         pywikibot.output('The bot is disabled')
         return
     super(LonelyPagesBot, self).run()

    def createGraph(self):
        """
        Create graph of the interwiki links.

        For more info see U{http://meta.wikimedia.org/wiki/Interwiki_graphs}
        """
        pywikibot.output(u'Preparing graph for %s'
                         % self.subject.originPage.title())
        # create empty graph
        self.graph = pydot.Dot()
        # self.graph.set('concentrate', 'true')

        self.octagon_sites = self._octagon_site_set()

        for page in self.subject.foundIn.keys():
            # a node for each found page
            self.addNode(page)
        # mark start node by pointing there from a black dot.
        firstLabel = self.getLabel(self.subject.originPage)
        self.graph.add_node(pydot.Node('start', shape='point'))
        self.graph.add_edge(pydot.Edge('start', firstLabel))
        for page, referrers in self.subject.foundIn.items():
            for refPage in referrers:
                self.addDirectedEdge(page, refPage)
        self.saveGraphFile()

def fillCaches(collectionqid):
    '''
    Build an ID cache so we can quickly look up the id's for property.
    Only return items in this ID cache for which we don't already have the Art UK artwork ID (P1679) link

    Build a second art uk -> Qid cache for items we don't have to process
    '''
    invcache = {}
    artukcache = {}
    sq = pywikibot.data.sparql.SparqlQuery()

    # FIXME: Do something with the collection qualifier
    query = u'SELECT ?item ?inv ?artukid WHERE { ?item wdt:P195 wd:%s . ?item wdt:P217 ?inv . OPTIONAL { ?item wdt:P1679 ?artukid } } ' % (collectionqid,)
    sq = pywikibot.data.sparql.SparqlQuery()
    queryresult = sq.select(query)

    for resultitem in queryresult:
        qid = resultitem.get('item').replace(u'http://www.wikidata.org/entity/', u'')
        if resultitem.get('artukid'):
            artukcache[resultitem.get('artukid')] = qid
        else:
            invcache[resultitem.get('inv')] = qid
    pywikibot.output(u'The query "%s" returned %s with and %s items without an ART UK work link' % (query,
                                                                                                    len(artukcache),
                                                                                                    len(invcache)))
    return (invcache,artukcache)

 def update_or_create_page(self, old_page, new_text):
     """
     Reads the current text of page old_page,
     compare it with new_text, prompts the user,
     and uploads the page
     """
     # Read the original content
     old_text = old_page.get()
     # Give the user some context
     if old_text != new_text:
         pywikibot.output(new_text)
     pywikibot.showDiff(old_text, new_text)
     # Get a decision
     prompt = u'Modify this page ?'
     # Did anything change ?
     if old_text == new_text:
         pywikibot.output(u'No changes necessary to %s' % old_page.title());
     else:
         if not self.acceptall:
             choice = pywikibot.input_choice(u'Do you want to accept these changes?',  [('Yes', 'Y'), ('No', 'n'), ('All', 'a')], 'N')
             if choice == 'a':
                 self.acceptall = True
         if self.acceptall or choice == 'y':
             # Write out the new version
             old_page.put(new_text, summary)

    def addQualifier(self, item, claim, qual):
        """
        Check if a qualifier is present at the given claim,
        otherwise add it

        Known issue: This will qualify an already referenced claim
            this must therefore be tested before

        param item: itemPage to check
        param claim: Claim to check
        param qual: Qualifier to check
        """
        # check if already present
        if self.hasQualifier(qual, claim):
            return False

        qClaim = self.make_simple_claim(qual.prop, qual.itis)

        try:
            claim.addQualifier(qClaim)  # writes to database
            pywikibot.output('Adding qualifier %s to %s in %s' %
                             (qual.prop, claim.getID(), item))
            return True
        except pywikibot.data.api.APIError, e:
            if e.code == u'modification-failed':
                pywikibot.output(u'modification-failed error: '
                                 u'qualifier to %s to %s in %s' %
                                 (qual.prop, claim.getID(), item))
                return False
            else:
                raise pywikibot.Error(
                    'Something went very wrong trying to add a qualifier: %s' %
                    e)

    def addReference(self, item, claim, ref):
        """Add a reference if not already present.

        param item: the item on which all of this happens
        param claim: the pywikibot.Claim to be sourced
        param ref: the WD.Reference to add
        """
        # check if any of the sources are already present
        # note that this can be in any of its references
        if ref is None:
            return False

        if any(self.hasRef(source.getID(), source.getTarget(), claim)
                for source in ref.source_test):
            return False

        try:
            claim.addSources(ref.get_all_sources())  # writes to database
            pywikibot.output('Adding reference claim to %s in %s' %
                             (claim.getID(), item))
            return True
        except pywikibot.data.api.APIError, e:
            if e.code == u'modification-failed':
                pywikibot.output(u'modification-failed error: '
                                 u'ref to %s in %s' % (claim.getID(), item))
                return False
            else:
                raise pywikibot.Error(
                    'Something went very wrong trying to add a source: %s' % e)

    def fillCache(self, collectionqid, idProperty, queryoverride=u'', cacheMaxAge=0):
        '''
        Query Wikidata to fill the cache of items we already have an object for
        '''
        result = {}
        if queryoverride:
            query = queryoverride
        else:
            query = u'CLAIM[195:%s] AND CLAIM[%s]' % (collectionqid.replace(u'Q', u''),
                                                      idProperty,)

        wd_queryset = wdquery.QuerySet(query)

        wd_query = wdquery.WikidataQuery(cacheMaxAge=cacheMaxAge)
        data = wd_query.query(wd_queryset, props=[str(idProperty),])

        if data.get('status').get('error')=='OK':
            expectedItems = data.get('status').get('items')
            props = data.get('props').get(str(idProperty))
            for prop in props:
                # FIXME: This will overwrite id's that are used more than once.
                # Use with care and clean up your dataset first
                result[prop[2]] = prop[0]

            if expectedItems==len(result):
                pywikibot.output('I now have %s items in cache' % expectedItems)
            else:
                pywikibot.output('I expected %s items, but I have %s items in cache' % (expectedItems, len(result),))

        return result

 def try_to_add(self):
     """Add current page in repo."""
     wd_data = set()
     for iw_page in self.iwlangs.values():
         try:
             wd_data.add(pywikibot.ItemPage.fromPage(iw_page))
         except pywikibot.NoPage:
             warning('Interwiki %s does not exist, skipping...' %
                     iw_page.title(asLink=True))
             continue
         except pywikibot.InvalidTitle:
             warning('Invalid title %s, skipping...' %
                     iw_page.title(asLink=True))
             continue
     if len(wd_data) != 1:
         warning('Interwiki conflict in %s, skipping...' %
                 self.current_page.title(asLink=True))
         return False
     item = list(wd_data).pop()
     if self.current_page.site.dbName() in item.sitelinks:
         warning('Interwiki conflict in %s, skipping...' %
                 item.title(asLink=True))
         return False
     output('Adding link to %s' % item.title())
     item.setSitelink(self.current_page)
     return item

    def login(self, retry=False):
        if not self.password:
            # As we don't want the password to appear on the screen, we set
            # password = True
            self.password = pywikibot.input(
                u'Password for user %(name)s on %(site)s (no characters will '
                u'be shown):' % {'name': self.username, 'site': self.site},
                password=True)
#        self.password = self.password.encode(self.site.encoding())

        pywikibot.output(u"Logging in to %(site)s as %(name)s"
                         % {'name': self.username, 'site': self.site})
        try:
            cookiedata = self.getCookie()
        except pywikibot.data.api.APIError as e:
            pywikibot.error(u"Login failed (%s)." % e.code)
            if retry:
                self.password = None
                return self.login(retry=True)
            else:
                return False
        self.storecookiedata(cookiedata)
        pywikibot.log(u"Should be logged in now")
##        # Show a warning according to the local bot policy
##   FIXME: disabled due to recursion; need to move this to the Site object after
##   login
##        if not self.botAllowed():
##            logger.error(
##                u"Username '%(name)s' is not listed on [[%(page)s]]."
##                 % {'name': self.username,
##                    'page': botList[self.site.family.name][self.site.code]})
##            logger.error(
##"Please make sure you are allowed to use the robot before actually using it!")
##            return False
        return True

    def treat_page(self):
        """Check page."""
        if (self.current_page.namespace() not in namespaces and
                not self.getOption('ignore_ns')):
            output('{page} is not in allowed namespaces, skipping'
                   .format(page=self.current_page.title(
                       asLink=True)))
            return False
        self.iwlangs = pywikibot.textlib.getLanguageLinks(
            self.current_page.text, insite=self.current_page.site)
        if not self.iwlangs:
            output('No interlanguagelinks on {page}'.format(
                page=self.current_page.title(asLink=True)))
            return False
        try:
            item = pywikibot.ItemPage.fromPage(self.current_page)
        except pywikibot.NoPage:
            item = None

        if item is None:
            item = self.try_to_add()
            if self.getOption('create') and item is None:
                item = self.create_item()

        self.current_item = item
        if item and self.getOption('clean'):
            self.clean_page()

 def run(self):
     """Start the bot."""
     template_image = i18n.translate(self.site,
                                     template_to_the_image)
     template_user = i18n.translate(self.site,
                                    template_to_the_user)
     summary = i18n.translate(self.site, comment, fallback=True)
     if not all([template_image, template_user, comment]):
         raise pywikibot.Error(u'This script is not localized for %s site.'
                               % self.site)
     self.summary = summary
     generator = pagegenerators.UnusedFilesGenerator(site=self.site)
     generator = pagegenerators.PreloadingGenerator(generator)
     for image in generator:
         if not image.exists():
             pywikibot.output(u"File '%s' does not exist (see bug 69133)."
                              % image.title())
             continue
         # Use fileUrl() and fileIsShared() to confirm it is local media
         # rather than a local page with the same name as shared media.
         if (image.fileUrl() and not image.fileIsShared() and
                 u'http://' not in image.text):
             if template_image in image.text:
                 pywikibot.output(u"%s done already"
                                  % image.title(asLink=True))
                 continue
             self.append_text(image, u"\n\n" + template_image)
             uploader = image.getFileVersionHistory().pop(0)['user']
             user = pywikibot.User(image.site, uploader)
             usertalkpage = user.getUserTalkPage()
             msg2uploader = template_user % {'title': image.title()}
             self.append_text(usertalkpage, msg2uploader)

    def showImageList(self, imagelist):
        """Print image list."""
        for i in range(len(imagelist)):
            image = imagelist[i]
            print("-" * 60)
            pywikibot.output(u"%s. Found image: %s"
                             % (i, image.title(asLink=True)))
            try:
                # Show the image description page's contents
                pywikibot.output(image.get())
                # look if page already exists with this name.
                # TODO: consider removing this: a different image of the same
                # name may exist on the target wiki, and the bot user may want
                # to upload anyway, using another name.
                try:
                    # Maybe the image is on the target site already
                    targetTitle = '%s:%s' % (self.targetSite.namespaces.FILE,
                                             image.title().split(':', 1)[1])
                    targetImage = pywikibot.Page(self.targetSite, targetTitle)
                    targetImage.get()
                    pywikibot.output(u"Image with this name is already on %s."
                                     % self.targetSite)
                    print("-" * 60)
                    pywikibot.output(targetImage.get())
                    sys.exit()
                except pywikibot.NoPage:
                    # That's the normal case
                    pass
                except pywikibot.IsRedirectPage:
                    pywikibot.output(
                        u"Description page on target wiki is redirect?!")

            except pywikibot.NoPage:
                break
        print("=" * 60)