def post(self):
    artwork_json = json.loads(self.request.get('json'))

    publish_date = (datetime.datetime
        .utcfromtimestamp(artwork_json['publishDate'] / 1000)
        .date())
    if FeaturedArtwork.all().filter('publish_date=', publish_date).get() != None:
      webapp2.abort(409, message='Artwork already exists for this date.')

    crop_tuple = tuple(float(x) for x in json.loads(self.request.get('crop')))

    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        crop_tuple,
        publish_date.strftime('%Y%m%d') + ' '
            + artwork_json['title'] + ' '
            + artwork_json['byline'])

    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']
    new_artwork = FeaturedArtwork(
        title=artwork_json['title'],
        byline=artwork_json['byline'],
        attribution=artwork_json['attribution'] if 'attribution' in artwork_json else None,
        image_url=new_image_url,
        thumb_url=new_thumb_url,
        details_url=artwork_json['detailsUri'],
        publish_date=publish_date)
    new_artwork.save()
    self.response.set_status(200)

  def post(self):
    artwork_json = json.loads(self.request.get('json'))
    crop_tuple = tuple(float(x) for x in json.loads(self.request.get('crop')))
    publish_date = (datetime.datetime
        .utcfromtimestamp(artwork_json['publishDate'] / 1000)
        .date())

    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        crop_tuple,
        publish_date.strftime('%Y%m%d') + ' '
            + artwork_json['title'] + ' '
            + artwork_json['byline'])

    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']
    new_artwork = FeaturedArtwork(
        title=artwork_json['title'],
        byline=artwork_json['byline'],
        image_url=new_image_url,
        thumb_url=new_thumb_url,
        details_url=artwork_json['detailsUri'],
        publish_date=publish_date)
    new_artwork.save()
    self.response.set_status(200)

  def get(self):
    ARTWORKS = json.loads(open(os.path.join(os.path.split(__file__)[0], 'lt-artworks.json')).read())

    # ARTWORKS = filter(lambda a: '_stars' in a and a['_stars'] >= 1, ARTWORKS)

    # Fetch latest 300 artworks (for blacklisting)
    latest_artworks = (FeaturedArtwork.all()
        .order('-publish_date')
        .fetch(300))

    # List dates for which artwork exists
    dates_with_existing_art = set(a.publish_date for a in latest_artworks)

    # List target dates that we want artwork for, but for which no artwork exists
    target_dates = [date.today() + timedelta(days=n) for n in range(-1, LOOKAHEAD_DAYS)]
    target_dates = [d for d in target_dates if d not in dates_with_existing_art]

    # Create a blacklist of keys to avoid repeats
    blacklist = set(artwork_key(a.details_url) for a in latest_artworks)

    logging.debug('starting blacklist size: %d' % len(blacklist))

    chosen_artworks = []

    for target_date in target_dates:
      # Pick from available artworks, excluding artwork in the blacklist
      random_artwork = None
      while True:
        if len(ARTWORKS) == 0:
          logging.error('Ran out of artworks to choose from, cannot continue')
          return

        random_artwork = random.choice(ARTWORKS)
        key = artwork_key(random_artwork['detailsUri'])
        if key not in blacklist:
          # Once chosen, remove it from the list of artworks to choose next
          ARTWORKS.remove(random_artwork)
          chosen_artworks.append(random_artwork)
          break

      target_details_url = str(random_artwork['detailsUri'])
      logging.debug('%(date)s: setting to %(url)s' % dict(url=target_details_url, date=target_date))

      # Store the new artwork
      if self.request.get('dry-run', '') != 'true':
        new_artwork = FeaturedArtwork(
            title=random_artwork['title'],
            byline=random_artwork['byline'],
            attribution=random_artwork['attribution'],
            image_url=random_artwork['imageUri'],
            thumb_url=random_artwork['thumbUri'],
            details_url=random_artwork['detailsUri'],
            publish_date=target_date)
        new_artwork.save()

    if self.request.get('output', '') == 'html':
      self.response.out.write(get_html(artworks_json=json.dumps(chosen_artworks)))

    # Finish up
    logging.debug('done')

    def post(self):
        artwork_json = json.loads(self.request.get("json"))

        publish_date = datetime.datetime.utcfromtimestamp(artwork_json["publishDate"] / 1000).date()
        if FeaturedArtwork.all().filter("publish_date=", publish_date).get() != None:
            webapp2.abort(409, message="Artwork already exists for this date.")

        crop_tuple = tuple(float(x) for x in json.loads(self.request.get("crop")))

        new_image_url, new_thumb_url = backroomarthelper.maybe_process_image(
            artwork_json["imageUri"],
            crop_tuple,
            publish_date.strftime("%Y%m%d") + " " + artwork_json["title"] + " " + artwork_json["byline"],
        )

        if not new_thumb_url and "thumbUri" in artwork_json:
            new_thumb_url = artwork_json["thumbUri"]
        new_artwork = FeaturedArtwork(
            title=artwork_json["title"],
            byline=artwork_json["byline"],
            attribution=artwork_json["attribution"] if "attribution" in artwork_json else None,
            image_url=new_image_url,
            thumb_url=new_thumb_url,
            details_url=artwork_json["detailsUri"],
            publish_date=publish_date,
        )
        new_artwork.save()
        self.response.set_status(200)

  def process_html(self, url, html):
    soup = BeautifulSoup(html)

    details_url = re.sub(r'#.+', '', url, re.I | re.S) + '?utm_source=Muzei&utm_campaign=Muzei'
    title = soup.find(itemprop='name').get_text()
    author = soup.find(itemprop='author').get_text()
    completion_year_el = soup.find(itemprop='dateCreated')
    byline = author + ((', ' + completion_year_el.get_text()) if completion_year_el else '')
    image_url = soup.find(id='paintingImage')['href']

    if not title or not author or not image_url:
      self.response.out.write('Could not parse HTML')
      self.response.set_status(500)
      return

    publish_date = (datetime.datetime
        .utcfromtimestamp(int(self.request.get('publishDate')) / 1000)
        .date())
    image_url, thumb_url = maybe_process_image(image_url,
        NO_CROP_TUPLE,
        publish_date.strftime('%Y%m%d') + ' ' + title + ' ' + byline)

    # create the artwork entry
    new_artwork = FeaturedArtwork(
        title=title,
        byline=byline,
        image_url=image_url,
        thumb_url=thumb_url,
        details_url=details_url,
        publish_date=publish_date)
    new_artwork.save()
    self.response.set_status(200)

    def process_html(self, url, html):
        soup = BeautifulSoup(html)

        details_url = re.sub(r"#.+", "", url, re.I | re.S) + "?utm_source=Muzei&utm_campaign=Muzei"
        title = soup.select("h1 span")[0].get_text()
        author = soup.find(itemprop="author").get_text()
        completion_year_el = soup.find(itemprop="dateCreated")
        byline = author + ((", " + completion_year_el.get_text()) if completion_year_el else "")
        image_url = soup.find(id="paintingImage")["href"]

        if not title or not author or not image_url:
            self.response.out.write("Could not parse HTML")
            self.response.set_status(500)
            return

        publish_date = datetime.datetime.utcfromtimestamp(int(self.request.get("publishDate")) / 1000).date()
        image_url, thumb_url = maybe_process_image(
            image_url, NO_CROP_TUPLE, publish_date.strftime("%Y%m%d") + " " + title + " " + byline
        )

        # create the artwork entry
        new_artwork = FeaturedArtwork(
            title=title,
            byline=byline,
            image_url=image_url,
            thumb_url=thumb_url,
            details_url=details_url,
            publish_date=publish_date,
        )
        new_artwork.save()
        self.response.set_status(200)

 def post(self):
   artwork_json = json.loads(self.request.get('json'))
   new_artwork = FeaturedArtwork(
       title=artwork_json['title'],
       byline=artwork_json['byline'],
       image_url=artwork_json['imageUri'],
       thumb_url=(artwork_json['thumbUri'] if 'thumbUri' in artwork_json else None),
       details_url=artwork_json['detailsUri'],
       publish_date=datetime.datetime
           .utcfromtimestamp(artwork_json['publishDate'] / 1000)
           .date())
   new_artwork.save()
   self.response.set_status(200)

    def post(self):
        id = long(self.request.get("id"))
        artwork_json = json.loads(self.request.get("json"))
        crop_tuple = tuple(float(x) for x in json.loads(self.request.get("crop")))
        target_artwork = FeaturedArtwork.get_by_id(id)
        if not target_artwork:
            self.response.set_status(404)
            return

        target_artwork.title = artwork_json["title"]
        target_artwork.byline = artwork_json["byline"]

        new_image_url, new_thumb_url = maybe_process_image(
            artwork_json["imageUri"],
            crop_tuple,
            target_artwork.publish_date.strftime("%Y%m%d") + " " + artwork_json["title"] + " " + artwork_json["byline"],
        )
        if not new_thumb_url and "thumbUri" in artwork_json:
            new_thumb_url = artwork_json["thumbUri"]

        target_artwork.image_url = new_image_url
        target_artwork.thumb_url = new_thumb_url
        target_artwork.details_url = artwork_json["detailsUri"]
        target_artwork.save()
        self.response.set_status(200)

 def move_artwork(self, artwork, publish_date, initial_artwork_id):
     # cascade moves
     current_artwork_at_date = FeaturedArtwork.all().filter("publish_date =", publish_date).get()
     if current_artwork_at_date and current_artwork_at_date.key().id() != initial_artwork_id:
         self.move_artwork(current_artwork_at_date, publish_date + datetime.timedelta(hours=24), initial_artwork_id)
     artwork.publish_date = publish_date
     artwork.save()

  def get(self):
    ARTWORKS = json.loads(open(os.path.join(os.path.split(__file__)[0], 'lt-artworks.json')).read())

    # Fetch latest 300 artworks (for blacklisting)
    latest_artworks = (FeaturedArtwork.all()
        .order('-publish_date')
        .fetch(300))

    # List dates for which artwork exists
    dates_with_existing_art = set(a.publish_date for a in latest_artworks)

    # List target dates that we want artwork for, but for which no artwork exists
    target_dates = [date.today() + timedelta(days=n) for n in range(-1, LOOKAHEAD_DAYS)]
    target_dates = [d for d in target_dates if d not in dates_with_existing_art]

    # Create a blacklist of keys to avoid repeats
    blacklist = set(artwork_key(a.details_url) for a in latest_artworks)

    self.response.out.write('starting blacklist size: %d<br>' % len(blacklist))

    for target_date in target_dates:
      # Pick from available artworks, excluding artwork in the blacklist
      random_artwork = None
      while True:
        random_artwork = random.choice(ARTWORKS)
        key = artwork_key(random_artwork['detailsUri'])
        if key not in blacklist:
          # Once chosen, add to the blacklist to avoid repeats within the lookahead
          blacklist.add(key)
          break

      target_details_url = str(random_artwork['detailsUri'])
      self.response.out.write('%(date)s: setting to <b>%(url)s</b><br>' % dict(url=target_details_url, date=target_date))

      # Store the new artwork
      new_artwork = FeaturedArtwork(
          title=random_artwork['title'],
          byline=random_artwork['byline'],
          attribution=random_artwork['attribution'],
          image_url=random_artwork['imageUri'],
          thumb_url=random_artwork['thumbUri'],
          details_url=random_artwork['detailsUri'],
          publish_date=target_date)
      new_artwork.save()

    # Finish up
    self.response.out.write('done<br>')

  def process_html(self, url, html):
    soup = BeautifulSoup(html)

    if re.search(r'wikiart.org', url, re.I):
      details_url = re.sub(r'#.+', '', url, re.I | re.S) + '?utm_source=Muzei&utm_campaign=Muzei'
      title = soup.select('h1 span')[0].get_text()
      author = soup.find(itemprop='author').get_text()
      completion_year_el = soup.find(itemprop='dateCreated')
      byline = author + ((', ' + completion_year_el.get_text()) if completion_year_el else '')
      image_url = soup.find(id='paintingImage')['href']
    elif re.search(r'metmuseum.org', url, re.I):
      details_url = re.sub(r'[#?].+', '', url, re.I | re.S) + '?utm_source=Muzei&utm_campaign=Muzei'
      title = soup.find('h2').get_text()
      author = unicode(soup.find(text='Artist:').parent.next_sibling).strip()
      author = re.sub(r'\s*\(.*', '', author)
      completion_year_el = unicode(soup.find(text='Date:').parent.next_sibling).strip()
      byline = author + ((', ' + completion_year_el) if completion_year_el else '')
      image_url = soup.find('a', class_='download').attrs['href']
    else:
      self.response.out.write('Unrecognized URL')
      self.response.set_status(500)
      return      

    if not title or not author or not image_url:
      self.response.out.write('Could not parse HTML')
      self.response.set_status(500)
      return

    publish_date = (datetime.datetime
        .utcfromtimestamp(int(self.request.get('publishDate')) / 1000)
        .date())
    image_url, thumb_url = maybe_process_image(image_url,
        NO_CROP_TUPLE,
        publish_date.strftime('%Y%m%d') + ' ' + title + ' ' + byline)

    # create the artwork entry
    new_artwork = FeaturedArtwork(
        title=title,
        byline=byline,
        image_url=image_url,
        thumb_url=thumb_url,
        details_url=details_url,
        publish_date=publish_date)
    new_artwork.save()

    self.response.set_status(200)
    self.response.out.write(json.dumps(artwork_dict(new_artwork)))

  def render_with_headers(self, callback):
    now = datetime.utcnow()
    headers = {}
    current = None

    # Get up to 5 artworks published earlier than 2 days from now, ordered by latest first
    latest_artworks = (FeaturedArtwork.all()
        .filter('publish_date <=', date.today() + timedelta(days=2))
        .order('-publish_date')
        .fetch(5))

    # Pick out the first artwork in that set that has actually been published
    for artwork in latest_artworks:
      if now >= datetime.combine(artwork.publish_date, START_TIME):
        current = artwork
        break

    ret_obj = dict()
    if current is not None:
      # Found the next featured artwork
      ret_obj = dict(
          title=current.title.strip(),
          byline=current.byline.strip(),
          imageUri=current.image_url,
          detailsUri=current.details_url)
      if current.thumb_url:
        ret_obj['thumbUri'] = current.thumb_url
      if current.attribution:
        ret_obj['attribution'] = current.attribution

      # The next update time is the next START_TIME
      next_start_time = datetime.combine(date.today(), START_TIME)
      while next_start_time < now:
        next_start_time += timedelta(hours=24)

      ret_obj['nextTime'] = _serialize_datetime(next_start_time + NEXT_PADDING)

      # Caches expire in an hour, but no later than the next start time minus padding
      cache_expire_time = min(
          now + MAX_HTTP_CACHE_AGE,
          next_start_time)
      expire_seconds = max(0, (cache_expire_time - now).total_seconds())

      # Note that this max-age header will be cached, so max-age may be off by the memcache
      # cache time which is set above to 60 seconds
      headers['Cache-Control'] = 'max-age=%d, must-revalidate, public' % expire_seconds
      headers['Expires'] = cache_expire_time.strftime('%a, %d %b %Y %H:%M:%S GMT')
      headers['Pragma'] = 'public'

    else:
      # Found no featured artwork; hopefully this is temporary; don't cache this response
      headers['Cache-Control'] = 'max-age=0, no-cache, no-store'
      headers['Pragma'] = 'no-cache'

    body = json.dumps(ret_obj, sort_keys=True)
    if callback:
      body = '%s(%s)' % (callback, body)

    return (body, headers)

 def render(self):
   start = datetime.date(day=1,
       month=int(self.request.get('month')) + 1,
       year=int(self.request.get('year')))
   start -= datetime.timedelta(weeks=2)
   queue = (FeaturedArtwork.all()
       .filter('publish_date >=', start)
       .order('publish_date')
       .fetch(1000))
   return json.dumps([artwork_dict(a) for a in queue])

 def render(self):
   queue = (FeaturedArtwork.all()
       .filter('publish_date >=', datetime.date.today() - datetime.timedelta(days=30))
       .order('publish_date')
       .fetch(1000))
   return json.dumps([dict(
       id=a.key().id(),
       title=a.title,
       byline=a.byline,
       imageUri=a.image_url,
       thumbUri=a.thumb_url,
       detailsUri=a.details_url,
       publishDate=date_to_timestamp(a.publish_date),)
       for a in queue])

  def post(self):
    id = long(self.request.get('id'))
    artwork_json = json.loads(self.request.get('json'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    target_artwork.title = artwork_json['title']
    target_artwork.byline = artwork_json['byline']
    target_artwork.image_url = artwork_json['imageUri']
    target_artwork.thumb_url = (artwork_json['thumbUri']
               if 'thumbUri' in artwork_json
               else (artwork_json['imageUri'] + '!BlogSmall.jpg'))
    target_artwork.details_url = artwork_json['detailsUri']
    target_artwork.save()
    self.response.set_status(200)

  def render(self, callback):
    now = datetime.datetime.utcnow()
    current = None

    # Get up to 5 artworks published earlier than 2 days from now, ordered by latest first
    latest_artworks = (FeaturedArtwork.all()
        .filter('publish_date <=', datetime.date.today() + datetime.timedelta(days=2))
        .order('-publish_date')
        .fetch(5))

    # Pick out the first artwork in that set that has actually been published
    for artwork in latest_artworks:
      if now >= datetime.datetime.combine(artwork.publish_date, START_TIME):
        current = artwork
        break

    ret_obj = dict()
    if current is not None:
      featured = dict(
          title=current.title,
          byline=current.byline,
          imageUri=current.image_url,
          detailsUri=current.details_url)
      if current.thumb_url:
        featured['thumbUri'] = current.thumb_url

      # The next update time is at START_TIME tomorrow
      next_time = datetime.datetime.combine(datetime.date.today() \
          + datetime.timedelta(days=1), START_TIME) + NEXT_PADDING
      featured['nextTime'] = _serialize_datetime(next_time)

      # Caches expire in an hour, but no later than the next start time minus 5 minutes
      cache_expire_time = min(
          datetime.datetime.now() + datetime.timedelta(hours=1),
          next_time - datetime.timedelta(minutes=5))
      expire_seconds = max(0, (cache_expire_time - now).total_seconds())
      self.response.headers['Cache-Control'] = 'max-age=%d, must-revalidate, public' % expire_seconds
      self.response.headers['Expires'] = cache_expire_time.strftime('%a, %d %b %Y %H:%M:%S GMT')

      ret_obj = featured

    s = json.dumps(ret_obj, sort_keys=True)
    if callback:
      return '%s(%s)' % (callback, s)
    else:
      return s

 def render(self):
   start = datetime.date(day=1,
       month=int(self.request.get('month')),
       year=int(self.request.get('year')))
   queue = (FeaturedArtwork.all()
       .filter('publish_date >=', start)
       .order('publish_date')
       .fetch(1000))
   return json.dumps([dict(
       id=a.key().id(),
       title=a.title,
       byline=a.byline,
       imageUri=a.image_url,
       thumbUri=a.thumb_url,
       detailsUri=a.details_url,
       publishDate=date_to_timestamp(a.publish_date),)
       for a in queue])

 def render(self):
     start = datetime.date(day=1, month=int(self.request.get("month")) + 1, year=int(self.request.get("year")))
     start -= datetime.timedelta(weeks=2)
     queue = FeaturedArtwork.all().filter("publish_date >=", start).order("publish_date").fetch(1000)
     return json.dumps(
         [
             dict(
                 id=a.key().id(),
                 title=a.title,
                 byline=a.byline,
                 imageUri=a.image_url,
                 thumbUri=a.thumb_url,
                 detailsUri=a.details_url,
                 publishDate=date_to_timestamp(a.publish_date),
             )
             for a in queue
         ]
     )

  def post(self):
    id = long(self.request.get('id'))
    artwork_json = json.loads(self.request.get('json'))
    target_artwork = FeaturedArtwork.get_by_id(id)
    if not target_artwork:
      self.response.set_status(404)
      return

    target_artwork.title = artwork_json['title']
    target_artwork.byline = artwork_json['byline']

    new_image_url, new_thumb_url = maybe_process_image(
        artwork_json['imageUri'],
        artwork_json['title'] + ' ' + artwork_json['byline'])
    if not new_thumb_url and 'thumbUri' in artwork_json:
      new_thumb_url = artwork_json['thumbUri']

    target_artwork.image_url = new_image_url
    target_artwork.thumb_url = new_thumb_url
    target_artwork.details_url = artwork_json['detailsUri']
    target_artwork.save()
    self.response.set_status(200)

  def get(self):
    # Fetch latest 1000 artworks
    latest_artworks = (FeaturedArtwork.all()
        .order('-publish_date')
        .fetch(1000))

    # List dates for which artwork exists
    dates_with_existing_art = set(a.publish_date for a in latest_artworks)

    # List target dates that we want artwork for, but for which no artwork exists
    target_dates = [date.today() + timedelta(days=n) for n in range(-1, 9)]
    target_dates = [d for d in target_dates if d not in dates_with_existing_art]

    for target_date in target_dates:
      self.response.out.write('looking for artwork for date ' + str(target_date) + '<br>')

      # Create a blacklist of the most recent 200 artwork
      # (don't want to repeat one of the last 200!)
      blacklist_artwork_keys = set(sanitized_artwork_key(a) for a in latest_artworks[:200])
      if len(blacklist_artwork_keys) < 5:
        blacklist_artwork_keys = set() # should never happen, but just in case of a reset

      # Pick from one of the oldest 500, excluding artwork in the blacklist
      random_artwork = None
      while True:
        random_artwork = random.choice(latest_artworks[500:])
        key = sanitized_artwork_key(random_artwork)
        if 'wikiart.org' in key or 'wikipaintings.org' in key or 'metmuseum.org' in key:
          if key not in blacklist_artwork_keys:
            break

      target_details_url = str(random_artwork.details_url)
      self.response.out.write('recycling ' + target_details_url + ' for date ' + str(target_date) + '<br>')

      backroomarthelper.add_art_from_external_details_url(
          target_date,
          target_details_url)

    self.response.out.write('done<br>')