def __get_offices(self, url, city_name=''):
     points = []
     page = PQ(get_url(url).decode('utf8'))
     time = None
     for item in map(PQ, page('#oo__content_value table tr:gt(0)')):
         if item('td').attr('colspan') == '3':
             continue
         point = Point()
         point.prov = self.uid
         point.type = TYPE_OFFICE
         point.name = normalize_text(item('td:eq(0)').text())
         point.address = normalize_address(city_name + item('td:eq(1) p:eq(0)').text())
         place = item('td:eq(1) p:eq(2)').text()
         if not place:
             place = item('td:eq(1) p:eq(1)').text()
         if place:
             point.place = normalize_text(place)
         new_time = item('td:eq(2)').text()
         if new_time:
             time = new_time
         point.time = normalize_time(time)
         point.check_information = CHECK_OFFICIAL
         if point.address in self.__addresses:
             point.lat, point.lng = self.__addresses[point.address]
             point.check_coordinates = CHECK_OFFICIAL
         else:
             warning_not_official_coordinates(point)
         points.append(point)
     return points

    def __parse_terminal(self, item):
        point = Point()
        point.prov = self.uid
        point.type = TYPE_TERMINAL

        city = normalize_text(item('td:eq(0)').text())
        address = normalize_text(item('td:eq(2)').text())
        point.address, point.place = split_address_place(u'г. %s, %s' % (city.title(), address))
        point.place = normalize_text(item('td:eq(1)').text())
        point.time = normalize_time(item('td:eq(3)').text())
        point.check_information = CHECK_OFFICIAL

        for lat, lng, type_id, description in self.__get_coordinates():
            if u'Минск' not in point.address or type_id != '2':
                continue
            for token in description.split():
                if token not in point.address:
                    break
            else:
                point.lat = lat
                point.lng = lng
                point.check_coordinates = CHECK_OFFICIAL
                break
        else:
            warning_not_official_coordinates(point)
        return point

    def __parse_base(self, item, city_name, point_type):
        point = Point()
        point.prov = self.uid
        point.type = point_type

        point.phones = [normalize_phone(item('.content_table table tbody tr:eq(0) td:eq(0) .office_phone').remove().text())]
        name_address_html = replace_br(item('.content_table table tbody tr:eq(0) td:eq(0)').remove().html(), ',')
        name, address = PQ(name_address_html).text().split(',', 1)
        point.name = normalize_text(name)
        point.address, point.place = self.__get_address(city_name, address)
        point.check_information = CHECK_OFFICIAL

        script_text = item('.ya_map script:eq(1)').text()
        for line in map(strip, script_text.splitlines()):
            if line.startswith('BX_GMapAddPlacemark('):
                lat_token = "'LAT':'"
                lat_start_index = line.find(lat_token) + len(lat_token)
                lat_end_index = line.find("'", lat_start_index)
                point.lat = line[lat_start_index:lat_end_index]
                lng_token = "'LON':'"
                lng_start_index = line.find(lng_token) + len(lng_token)
                lng_end_index = line.find("'", lng_start_index)
                point.lng = line[lng_start_index:lng_end_index]
                point.check_coordinates = CHECK_OFFICIAL
                break
        else:
            warning_not_official_coordinates(point)
        return point

 def __parse_base(self, item, city, point_type):
     point = Point()
     point.prov = self.uid
     point.type = point_type
     point.name = normalize_text(item('.b-map-side>h5').text())
     point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('.b-map-side>p span:eq(0)').text()))
     coordinates = item('.b-map-side>p span:eq(1)').text()
     if coordinates:
         point.lat, point.lng = map(strip, coordinates.split(','))
     text_html = replace_br(item('.b-map-side-more').html(), ';;;')
     time_items = []
     for sub_item in map(normalize_text, PQ(text_html).text().split(';;;')):
         if not sub_item:
             continue
         if sub_item.startswith(u'Телефон:'):
             point.phones = normalize_phones(sub_item[len(u'Телефон:')].split(','))
             continue
         time_items.append(sub_item)
     point.time = normalize_time(', '.join(time_items))
     point.check_information = CHECK_OFFICIAL
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point

    def __parse_base_office_exchange(self, item, point_type, name_keywords):
        point = Point()
        point.prov = self.uid
        point.type = point_type
        point.name = normalize_text(item('th:eq(0) a:eq(0)').text())
        if not point.name.startswith(name_keywords):
            return None

        city = normalize_text(item('td:eq(1)').text())
        address = normalize_text(item('td:eq(2)').text())
        point.address, point.place = split_address_place(u'г. %s, %s' % (city, address))
        point.check_information = CHECK_OFFICIAL

        for lat, lng, type_id, description in self.__get_coordinates():
            if u'Минск' not in point.address or type_id != '1':
                continue
            for token in description.split():
                if token not in point.address and token not in point.name:
                    break
            else:
                point.lat = lat
                point.lng = lng
                point.check_coordinates = CHECK_OFFICIAL
                break
        else:
            warning_not_official_coordinates(point)
        return point

 def post(self):
     user = users.get_current_user()
     if user == None:
         self.response.out.write()
     else:
         lat = float(self.request.get("lat"))
         lon = float(self.request.get("lon"))
         hash = geohash.encode(lat, lon)
         point = Point()
         point.geohash = hash
         point.owner = users.get_current_user()
         point.put()
         template_values = {"method": "post"}
         self.response.headers["Content-Type"] = "application/json"
         self.response.out.write('{result:"OK", geohash:"' + hash + '"}')

 def __parse_office_main(self, coordinates):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_OFFICE
     point.name = u'Центральный офис'
     point.address = u'г. Минск, ул. В.Хоружей, 31а'
     point.phones = [u'+375172899090', u'+375172899292']
     point.time = u'пн-чт: 08:30-17:30, пт: 08:30-16:15, перерыв: 12:30-13:15, сб, вс: выходной'
     point.check_information = CHECK_OFFICIAL
     point.lat, point.lng = self.__get_point_coordinate(point.address, coordinates)
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point

 def __parse_terminal(self, item):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_TERMINAL
     point.address, point.place = split_address_place(item('td:eq(2)').text())
     point.place = normalize_text(item('td:eq(1)').text())
     point.currency = map(strip, item('td:eq(4)').text().split(','))
     if point.currency:
         point.deposit = True
     else:
         point.deposit = False
     point.time = normalize_time(item('td:eq(3)').text())
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point

 def __parse_atm(self, item, city, coordinates):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_ATM
     point.address = normalize_address(u'%s, %s' % (city, item('td:eq(2)').text()))
     point.place = normalize_text(item('td:eq(1)').text())
     point.currency = map(strip, item('td:eq(4)').text().replace('EURO', 'EUR').split(','))
     point.time = normalize_time(item('td:eq(3)').text())
     point.check_information = CHECK_OFFICIAL
     point.lat, point.lng = self.__get_point_coordinate(point, coordinates)
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point

 def __parse_terminal(self, item):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_TERMINAL
     point.name = normalize_text(item('td:eq(0)').text())
     point.address, point.place = split_address_place(item('td:eq(1)').text())
     point.place = point.name
     point.time = normalize_time(item('td:eq(2)').text())
     point.deposit = normalize_text(item('td:eq(3)').text()).lower() == u'есть'
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point

 def get(self):
     user = users.get_current_user()
     if user == None:
         self.redirect(users.create_login_url(self.request.uri))
     userPoints = Point.gql("WHERE owner = :1 ORDER BY created DESC ", users.get_current_user())
     template_values = {"method": "get", "points": userPoints}
     self.response.out.write(template.render("templates/history.html", template_values))

 def post(self):
     a, d = self.request.get('author'), self.request.get('description')  # error: missing either key
     try:
         schedString = self.request.get('schedulestring')   # error: missing parameter
         jsonSched = simplejson.loads(schedString)          # error: bad json
     
         s = Schedule(auth = a, desc = d)
         s.put()                                            # error: some puts succeed while others fail
         for pt in jsonSched["points"]:                     # error: not an object, or no "points" key
             p = Point(sched = s, location = map(int, pt), phase = ['R', 'R'])  # error:  setting all phases to RR
             p.put()                                        # error: some puts succeed while others fail
     # schedule should be post-ed as json string
     # so parse the string into Schedule's and Point's, then save it 
     #   (check out previous versions of scheduler.py for examples of saving)
         self.response.out.write(simplejson.dumps({'success': 'Schedule was successfully saved!'})) # maybe add the schedule's id?
     except Exception, e:
         self.response.out.write(simplejson.dumps({'error': e.message}))

 def get_offices(self):
     points = []
     items_tree = ET.fromstring(get_url(self.__offices_xml_url))
     for item in items_tree.iter('item'):
         point = self.__parse_office(item)
         if point:
             points.append(point)
     page = PQ(get_url(self.__regional_offices_page_url))
     point = None
     for item in map(PQ, page('#content_internal span:eq(0)').children()):
         if item[0].tag not in self.__regional_offices_tags:
             continue
         if item[0].tag == 'h2':
             point = Point()
             point.prov = self.uid
             point.type = TYPE_OFFICE
             point.name = trim_spaces_and_commas(normalize_text(item.text()))
             point.check_information = CHECK_OFFICIAL
             continue
         if not point:
             continue
         item_html = replace_br(item.html(), ';;;')
         sub_items = PQ(item_html).text().split(';;;')
         point.address, point.place = split_address_place(sub_items[0])
         for sub_item in map(normalize_text, sub_items[1:]):
             if sub_item.startswith(u'т.ф.:'):
                 point.phone = normalize_phones(sub_item[len(u'т.ф.:'):].split(','))
         warning_not_official_coordinates(point)
         points.append(point)
         point = None
     return points

 def __parse_atm(self, item):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_ATM
     point.name = normalize_text(item('td:eq(0)').text())
     point.address, point.place = split_address_place(item('td:eq(1)').text())
     point.place = point.name
     point.time = normalize_time(item('td:eq(2)').text())
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point

 def __parse_atm(self, item):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_ATM
     point.address = normalize_address(u'г. %s' % item('td:eq(0)').text())
     point.place = normalize_text(item('td:eq(1)').text())
     point.time = normalize_time(item('td:eq(2)').text())
     point.currency = map(strip, item('td:eq(3)').text().split(','))
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point

 def get_atms(self):
     points = []
     page = PQ(get_url(self.__parse_list_atm_url).decode('utf8'))
     for item in map(PQ, page('#oo__content_value table tr:gt(0)')):
         point = Point()
         point.prov = self.uid
         point.type = TYPE_ATM
         point.address = normalize_address(item('td:eq(0) p:eq(0)').text())
         point.place = normalize_text(item('td:eq(1)').text())
         point.time = normalize_time(item('td:eq(2)').text())
         point.currency = map(self.__get_currency, item('td:eq(3) p'))
         point.check_information = CHECK_OFFICIAL
         if point.address in self.__addresses:
             point.lat, point.lng = self.__addresses[point.address]
             point.check_coordinates = CHECK_OFFICIAL
         else:
             warning_not_official_coordinates(point)
         points.append(point)
     return points

 def __parse_base_office_exchange(self, item):
     point = Point()
     point.prov = self.uid
     point.name = normalize_text(item('td:eq(1)').text())
     point.address, point.place = split_address_place(item('td:eq(2)').text())
     point.time = normalize_time(item('td:eq(3)').text())
     point.phones = normalize_phones(item('td:eq(4)').text().split(','))
     point.check_information = CHECK_OFFICIAL
     return point

def parse(filename,filepointer):
    rows = filepointer.readlines()
    airspaceFile = AirspaceFile(name=filename, importDate=datetime.now())
    counter = 0
    previousLine = ''
    for line in rows:
	#logger.debug(line)
        identifier = line[:2]
        bareline = line[3:].replace('\r\n','')
        if(re.match("^[A-Za-z]",identifier)):
            if identifier == 'AC':
                airspace = Airspace(type=AIRSPACE_CLASSES[bareline],description=previousLine[14:].replace('\r\n',''))
                airspaceFile.airspaces.append(airspace)
                counter = 0
            elif identifier == 'AN':
                if bareline.startswith('BERGBAHN'):
                    airspace.subtype = 'CABLECAR'
                    airspace.name = bareline[9:]
                elif bareline.startswith('KABEL'):
                    airspace.subtype = 'CABLE'
                    airspace.name = bareline[6:]
                elif bareline.startswith('SCHUTZ'):
                    airspace.subtype = 'WILDLIFE_PROTECTION'
                    airspace.name = bareline[7:]
                else:
                    airspace.name = bareline
            elif identifier == 'AH':
                #airspace.ceiling = int(bareline.replace('GND', '0').replace('ft', '').strip())
                airspace.ceiling = bareline
            elif identifier == 'AL':
                #airspace.floor = int(bareline.replace('GND', '0').replace('ft', '').strip())
                airspace.floor = bareline
            elif identifier == 'DP':
               point = Point(index=counter)
               point.latitude = bareline[:8]
               point.latitude_dec = dms2dec(point.latitude[:2],point.latitude[3:5],point.latitude[6:])
               point.longitude = bareline[11:20]
               point.longitude_dec = dms2dec(point.longitude[:3],point.longitude[4:6],point.longitude[7:])
               airspace.points.append(point)
               counter += 1
        previousLine = line
    return airspaceFile

 def __parse_base(self, item, city_name, point_type):
     point = Point()
     point.prov = self.uid
     point.type = point_type
     point.address, point.place = self.__parse_address(city_name, item('td:eq(0) a').text())
     point.check_information = CHECK_OFFICIAL
     point.lat = item('td:eq(0) .item_coords .coord1').text()
     point.lng = item('td:eq(0) .item_coords .coord2').text()
     if point.lat and point.lng:
         point.check_coordinates = CHECK_OFFICIAL
     else:
         warning_not_official_coordinates(point)
     return point

 def __parse_exchange(self, item, city):
     point = Point()
     point.prov = self.uid
     point.type = TYPE_EXCHANGE
     point.name = normalize_text(item('td:eq(0)').text())
     point.address, point.place = split_address_place(u'г. %s, %s' % (city, item('td:eq(1)').text()))
     if len(item('td')) == 4:
         point.time = normalize_time(item('td:eq(2)').text())
     else:
         point.time = normalize_time(item('td:eq(2)').text().split(u'Операции:')[0])
     point.check_information = CHECK_OFFICIAL
     warning_not_official_coordinates(point)
     return point