def patch_hyphen(srcdir, dstdir, copy_unchanged=True):
  """Add hyphen-minus glyphs to fonts that need it.

  This is to enable languages to be hyphenated properly,
  since Minikin's itemizer currently shows tofus if an
  automatically hyphenated word is displated in a font
  that has neither HYPHEN nor HYPHEN-MINUS.

  The list of font names comes from LANG_TO_SCRIPT in
  tools/font/fontchain_lint.py.

  (In practice only U+002D HYPHEN-MINUS is added, since Noto LGC fonts
  don't have U+2010 HYPHEN.)

  Bug: 21570828"""

  # Names of fonts for which Android requires a hyphen.
  # This list omits Japanese and Korean.
  script_names = [
      'Armenian', 'Ethiopic', 'Bengali', 'Gujarati', 'Devanagari',
      'Kannada', 'Malayalam', 'Oriya', 'Gurmukhi', 'Tamil', 'Telugu']

  HYPHENS = {0x002D, 0x2010}

  for sn in script_names:
    globexp = path.join(srcdir, 'Noto*%s-*.ttf' % sn)
    fonts = glob.glob(globexp)
    if not fonts:
      raise ValueError('could not match ' + globexp)
    fonts = [path.basename(f) for f in fonts]
    for font_name in fonts:
      lgc_font_name = font_name.replace(sn, '')

      font_file = path.join(srcdir, font_name)
      lgc_font_file = path.join(srcdir, lgc_font_name)

      chars_to_add = (
          (HYPHENS - coverage.character_set(font_file))
          & coverage.character_set(lgc_font_file))

      if chars_to_add:
        print 'patch hyphens', font_name
        merger.merge_chars_from_bank(
            path.join(srcdir, font_name),
            path.join(srcdir, lgc_font_name),
            path.join(srcdir, font_name),
            chars_to_add)
      else:
        if copy_unchanged:
          shutil.copy2(
              path.join(srcdir,font_name), path.join(dstdir, font_name))
          print '%s already has hyphens, copying' % font_name
        else:
          print '%s already has hyphens' % font_name

 def test_lack_of_unassigned_chars(self):
     """Tests that unassigned characters are not in the fonts."""
     for font in self.fonts:
         charset = coverage.character_set(font)
         self.assertNotIn(0x2072, charset)
         self.assertNotIn(0x2073, charset)
         self.assertNotIn(0x208F, charset)

 def test_lack_of_arrows_and_combining_keycap(self):
     """Tests that arrows and combining keycap are not in the fonts."""
     for font in self.fonts:
         charset = coverage.character_set(font)
         self.assertNotIn(0x20E3, charset)  # COMBINING ENCLOSING KEYCAP
         self.assertNotIn(0x2191, charset)  # UPWARDS ARROW
         self.assertNotIn(0x2193, charset)  # DOWNWARDS ARROW

    def test_inclusion(self):
        """Tests for characters which should be included."""

        for font in self.fonts:
            charset = coverage.character_set(font)
            for char in self.include:
                self.assertIn(char, charset)

    def test_exclusion(self):
        """Tests that characters which should be excluded are absent."""

        for font in self.fonts:
            charset = coverage.character_set(font)
            for char in self.exclude:
                self.assertNotIn(char, charset)

 def test_inclusion_of_sound_recording_copyright(self):
     """Tests that sound recording copyright symbol is in the fonts."""
     for font in self.fonts:
         charset = coverage.character_set(font)
         self.assertIn(
             0x2117, charset,  # SOUND RECORDING COPYRIGHT
             'U+2117 not found in %s.' % font_data.font_name(font))

def main(argv):
    """Fix all the fonts given in the command line.
    
    If they are Lao fonts, make sure they have ZWSP and dotted circle. If they
    are Khmer fonts, make sure they have ZWSP, joiners, and dotted circle."""

    for font_name in argv[1:]:
        if 'Khmer' in font_name:
            script = 'Khmr'
        elif 'Lao' in font_name:
            script = 'Laoo'
        needed_chars = set(opentype_data.SPECIAL_CHARACTERS_NEEDED[script])

        lgc_font_name = (
            os.path.basename(font_name).replace('Khmer', '').replace('Lao', ''))
        lgc_font_name = os.path.join(_UNHINTED_FONTS_DIR, lgc_font_name)

        font_charset = coverage.character_set(font_name)
        missing_chars = needed_chars - font_charset
        if missing_chars:
            merge_chars_from_bank(
                font_name,
                lgc_font_name,
                os.path.dirname(font_name)+'/new/'+os.path.basename(font_name),
                missing_chars)

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '-f', '--font',
      help='font whose character map to restrict text to',
      required=True)
  parser.add_argument(
      '-t', '--text',
      help='initial text, prepend @ to read from file')

  args = parser.parse_args()
  if args.text:
    if args.text[0] == '@':
      with codecs.open(args.text[1:], 'r', 'utf-8') as f:
        text = f.read()
    else:
      text = args.text
  else:
    text = ''

  if args.font:
    charset = coverage.character_set(args.font)
    name_map = _get_char_names(charset)
    text = _build_text(name_map, text)
    print 'text: ' + text
  else:
    charset = None

 def test_non_inclusion_of_other_pua(self):
     """Tests that there are not other PUA characters except legacy ones."""
     for font in self.fonts:
         charset = coverage.character_set(font)
         pua_chars = {
             char for char in charset
             if 0xE000 <= char <= 0xF8FF or 0xF0000 <= char <= 0x10FFFF}
         self.assertTrue(pua_chars <= self.LEGACY_PUA)

    def test_sanity(self):
        """Test basic sanity of the method."""
        font_file = tempfile.NamedTemporaryFile()
        font = make_font('')
        font.save(font_file.name)
        charset = coverage.character_set(font_file.name)

        self.assertTrue(ord(' ') in charset)
        self.assertTrue(ord('A') in charset)
        self.assertFalse(0x10B00 in charset)

def get_cps_from_files(paths):
  """Return a tuple of the cps and the paths that we actually read."""
  cps = set()
  new_paths = set()
  for f in paths:
    ext = path.splitext(f)[1]
    if ext not in ['.otf', '.ttf']:
      continue
    cps |= coverage.character_set(f)
    new_paths.add(f)
  return cps, sorted(new_paths)

def get_families(fonts):
  """Group fonts into families, group by hinted and unhinted, select representative."""
  families = {}
  all_keys = set([font.key for font in fonts])
  for key in all_keys:
    members = {font for font in fonts
               if font.key == key and font.variant != 'UI' and font.fmt in {'ttf', 'otf'}}

    if not members:
      mbrs = {font for font in fonts if font.key == key}
      raise ValueError("no members for %s from %s" % (key, [f.filepath for f in mbrs]))

    hinted_members = []
    unhinted_members = []
    rep_member = None
    for font in members:
      if font.hint_status == 'hinted':
        hinted_members.append(font)
      else:
        unhinted_members.append(font)
      if not rep_member:
        if font.weight == 'Regular' and font.slope is None:
          # We assume here that there's no difference between a hinted or unhinted
          # rep_member in terms of what we use it for
          rep_member = font

    if not rep_member:
      raise ValueError('Family %s does have a representative font.' % key)

    if hinted_members and not len(hinted_members) in [1, 2, 4, 7]:
      raise ValueError('Family %s has %d hinted_members (%s)' % (
          key, len(hinted_members), [font.name for font in hinted_members]))

    if unhinted_members and not len(unhinted_members) in [1, 2, 4, 7]:
      raise ValueError('Family %s has %d unhinted_members (%s)' % (
          key, len(unhinted_members), [font.name for font in unhinted_members]))

    if hinted_members and unhinted_members and len(hinted_members) != len(unhinted_members):
      raise ValueError('Family %s has %d hinted members but %d unhinted memberts' % (
          key, len(hinted_members), len(unhinted_members)))

    name = get_font_family_name(rep_member.filepath)

    if rep_member.fmt in {'ttf', 'otf'}:
      charset = coverage.character_set(rep_member.filepath)
    else:
      charset = NotImplemented

    families[key] = Family(name, rep_member, charset, hinted_members, unhinted_members)

  return families

def get_families(fonts):
  """Group fonts into families, separate into hinted and unhinted, select
  representative."""

  family_id_to_fonts = collections.defaultdict(set)
  families = {}
  for font in fonts:
    family_id = noto_font_to_family_id(font)
    family_id_to_fonts[family_id].add(font)

  for family_id, fonts in family_id_to_fonts.iteritems():
    hinted_members = []
    unhinted_members = []
    rep_member = None
    rep_backup = None  # used in case all fonts are ttc fonts
    for font in fonts:
      if font.is_hinted:
        hinted_members.append(font)
      else:
        unhinted_members.append(font)
      if not rep_member:
        if font.weight == 'Regular' and font.slope is None and not (
            font.is_cjk and font.is_mono) and not font.is_UI:
          # We assume here that there's no difference between a hinted or
          # unhinted rep_member in terms of what we use it for.  The other
          # filters are to ensure the fontTools font name is a good stand-in
          # for the family name.
          if font.fmt == 'ttc' and not rep_backup:
            rep_backup = font
          else:
            rep_member = font

    rep_member = rep_member or rep_backup
    if not rep_member:
      raise ValueError(
          'Family %s does not have a representative font.' % family_id)

    name = get_font_family_name(rep_member.filepath)

    if rep_member.fmt in {'ttf', 'otf'}:
      charset = coverage.character_set(rep_member.filepath)
    else:
      # was NotImplemented, but bool(NotImplemented) is True
      charset = None

    families[family_id] = NotoFamily(
        name, family_id, rep_member, charset, hinted_members, unhinted_members)

  return families

def main():
    """Checkes the coverage of all Roboto fonts."""
    with open('res/char_requirements.tsv') as char_reqs_file:
        char_reqs_data = char_reqs_file.read()

    # The format of the data to be parsed is like the following:
    # General Punctuation\t2000..206F\t111\t35\t54\t0\tEverything except 2028..202E, 2060..2064, and 2066..206F
    # Currency Symbols\t20A0..20CF\t29\t5\t24\t1\t
    required_set = set()
    for line in char_reqs_data.split('\n'):
        if line.startswith('#'):  # Skip comment lines
            continue
        line = line.split('\t')
        if not line[0]:
            continue  # Skip the first line and empty lines
        block_range = line[1]
        full_coverage_required = (line[5] == '1')
        exceptions = line[6]
        required_set.update(
            _find_required_chars(block_range,
                                 full_coverage_required,
                                 exceptions))

    # Skip Unicode 8.0 characters
    required_set = {ch for ch in required_set
                    if float(unicode_data.age(ch)) <= 7.0}

    # Skip ASCII and C1 controls
    required_set -= set(range(0, 0x20) + range(0x7F, 0xA0))

    missing_char_found = False
    for font in load_fonts():
        font_coverage = coverage.character_set(font)
        missing_chars = required_set - font_coverage
        if missing_chars:
            missing_char_found = True
            font_name = font_data.font_name(font)
            print 'Characters missing from %s:' % font_name
            for char in sorted(missing_chars):
                _print_char(char)
            print

    if missing_char_found:
        sys.exit(1)

def get_families(fonts):
  """Group fonts into families, separate into hinted and unhinted, select representative."""
  family_id_to_fonts = collections.defaultdict(set)
  families = {}
  for font in fonts:
    family_id = noto_font_to_family_id(font)
    family_id_to_fonts[family_id].add(font)

  for family_id, fonts in family_id_to_fonts.iteritems():
    hinted_members = []
    unhinted_members = []
    rep_member = None
    for font in fonts:
      if font.is_hinted:
        hinted_members.append(font)
      else:
        unhinted_members.append(font)
      if not rep_member:
        if font.weight == 'Regular' and font.slope is None and not font.is_mono:
          # We assume here that there's no difference between a hinted or unhinted
          # rep_member in terms of what we use it for.  The other filters are to ensure
          # the fontTools font name is a good stand-in for the family name.
          rep_member = font

    if not rep_member:
      raise ValueError('Family %s does not have a representative font.' % family_id)

    if hinted_members and not len(hinted_members) in [1, 2, 4, 7, 8, 9]: # 9 adds the two Mono variants
      raise ValueError('Family %s has %d hinted_members (%s)' % (
          family_id, len(hinted_members), [path.basename(font.filepath) for font in hinted_members]))

    if unhinted_members and not len(unhinted_members) in [1, 2, 4, 8]:
      raise ValueError('Family %s has %d unhinted_members (%s)' % (
          family_id, len(unhinted_members), [
              path.basename(font.filepath) for font in unhinted_members]))

    if hinted_members and unhinted_members and len(hinted_members) != len(unhinted_members):
      # Let's not consider this an error for now.  Just drop the members with the higher number
      # of fonts, assuming it's a superset of the fonts in the smaller set, so that the fonts
      # we provide and display are available to all users.  This means website users will not be
      # able to get these fonts via the website.
      #
      # We'll keep the representative font and not try to change it.
      print 'Family %s has %d hinted members but %d unhinted memberts' % (
          family_id, len(hinted_members), len(unhinted_members))
      if len(hinted_members) < len(unhinted_members):
        unhinted_members = []
      else:
        hinted_members = []

    name = get_font_family_name(rep_member.filepath)

    if rep_member.fmt in {'ttf', 'otf'}:
      charset = coverage.character_set(rep_member.filepath)
    else:
      charset = NotImplemented

    families[family_id] = NotoFamily(
        name, family_id, rep_member, charset, hinted_members, unhinted_members)

  return families

 def setUp(self):
     self.font_files, _ = self.loaded_fonts
     charset = coverage.character_set(self.font_files[0])
     self.marks_to_test = [char for char in charset
                           if unicode_data.combining(char) == 230]
     self.advance_cache = {}

 def test_inclusion_of_legacy_pua(self):
     """Tests that legacy PUA characters remain in the fonts."""
     for font in self.fonts:
         charset = coverage.character_set(font)
         for char in self.LEGACY_PUA:
             self.assertIn(char, charset)

def find_fonts():
    font_name_regexp = re.compile(
        '(NotoSans|NotoSerif|NotoNaskh|NotoKufi|Arimo|Cousine|Tinos)'
        '(Mono)?'
        '(.*?)'
        '(UI|Eastern|Estrangela|Western)?'
        '-'
        '(|Black|Bold|DemiLight|Light|Medium|Regular|Thin)'
        '(Italic)?'
        '(-Windows)?'
        '.[ot]t[cf]')

    unicode_data.load_data()

    for directory in [path.join(INDIVIDUAL_FONT_DIR, 'hinted'),
                      path.join(INDIVIDUAL_FONT_DIR, 'unhinted'),
                      path.join(FONT_DIR, 'alpha'),
                      CJK_DIR]:
        for filename in os.listdir(directory):
            match = font_name_regexp.match(filename)
            if match:
                family, mono, script, variant, weight, style, platform = match.groups()
            elif filename == 'NotoNastaliqUrduDraft.ttf':
                family = 'NotoNastaliq'
                script = 'Aran'  # Arabic Nastaliq
                weight = ''
                style = variant = platform = None
            else:
                if not (
                    filename == 'NotoSansCJK.ttc' or  # All-comprehensive CJK
                    filename.endswith('.ttx') or
                    filename.startswith('README.') or
                    filename in ['COPYING', 'LICENSE', 'NEWS', 'HISTORY']):
                    raise ValueError("unexpected filename in %s: '%s'." %
                                     (directory, filename))
                continue

            if directory == CJK_DIR:
                license_type = 'sil'
            else:
                license_type = 'apache'

            if mono:
                # we don't provide the Mono CJK on the website
                continue
            if script == "Historic":
                # we don't provide this either
                continue

            if family in {'Arimo', 'Cousine', 'Tinos'}:
                continue  # Skip these three for the website

            if family.startswith('Noto'):
                family = family.replace('Noto', 'Noto ')

            if weight == '':
                weight = 'Regular'

            assert platform is None

            if script == '':  # LGC
                supported_scripts.update({'Latn', 'Grek', 'Cyrl'})
            elif script == 'Aran':
                supported_scripts.add(script)
            elif script in {'JP', 'KR', 'SC', 'TC', 'CJK'}:
                continue  # Skip unified or old CJK fonts
            else:
                script = convert_to_four_letter(script)
                supported_scripts.add(script)

            file_path = path.join(directory, filename)
            if filename.endswith('.ttf') or filename.endswith('.otf'):
                charset = coverage.character_set(file_path)
            else:
                charset = NotImplemented

            if directory == CJK_DIR:
                hint_status = 'hinted'
            elif directory.endswith('alpha'):
                hint_status = 'unhinted'
            else:
                hint_status = path.basename(directory)
            assert hint_status in ['hinted', 'unhinted']

            key = family.replace(' ', '-')
            if script:
                key += '-' + script
            if variant not in {None, 'UI'}:
                key += '-' + variant
            key = key.lower()

            font = Font(file_path, hint_status, key,
                        family, script, variant, weight, style, platform,
                        charset, license_type)
            all_fonts.append(font)