/**
  * Tries to convert the given HTML into a plain text format - best suited for
  * e-mail display, etc.
  *
  * <p>In particular, it tries to maintain the following features:
  * <ul>
  *   <li>Links are maintained, with the 'href' copied over
  *   <li>Information in the &lt;head&gt; is lost
  * </ul>
  *
  * @param string $html the input HTML
  * @return string the HTML converted, as best as possible, to text
  * @throws Html2TextException if the HTML could not be loaded as a {@link DOMDocument}
  */
 static function convert($html)
 {
     // replace &nbsp; with spaces
     $html = str_replace("&nbsp;", " ", $html);
     $html = str_replace(" ", " ", $html);
     if (static::isOfficeDocument($html)) {
         // remove office namespace
         $html = str_replace(array("<o:p>", "</o:p>"), "", $html);
     }
     $html = static::fixNewlines($html);
     if (mb_detect_encoding($html, "UTF-8", true)) {
         $html = mb_convert_encoding($html, "HTML-ENTITIES", "UTF-8");
     }
     $doc = new \DOMDocument();
     if (!$doc->loadHTML($html)) {
         throw new Html2TextException("Could not load HTML - badly formed?", $html);
     }
     if (static::isOfficeDocument($html)) {
         // remove office namespace
         $doc = static::fixMSEncoding($doc);
     }
     $output = static::iterateOverNode($doc);
     // remove leading and trailing spaces on each line
     $output = preg_replace("/[ \t]*\n[ \t]*/im", "\n", $output);
     $output = preg_replace("/ *\t */im", "\t", $output);
     // remove unnecessary empty lines
     $output = preg_replace("/\n\n\n*/im", "\n\n", $output);
     // remove leading and trailing whitespace
     $output = trim($output);
     return $output;
 }

 function get_content($url)
 {
     require_once XOOPS_ROOT_PATH . '/class/snoopy.php';
     if ($url) {
         $this->url = preg_replace('|/+$|', '', $url);
         $this->url = preg_replace('|#.*$|', '', $this->url);
     }
     $snoopy = new Snoopy();
     if ($snoopy->fetch($this->url)) {
         $tb_contents = $snoopy->results;
         if (function_exists('mb_detect_encoding')) {
             $this->charset = mb_detect_encoding($tb_contents, "auto");
         }
         if (preg_match_all('#<rdf:RDF[^>]*>(.*?)</rdf:RDF>#si', $tb_contents, $matches, PREG_PATTERN_ORDER)) {
             foreach ($matches[1] as $tb_body) {
                 //thanks  kenken
                 $this->xml_parts[] = preg_replace('|dc:description=\\"[^\\"]*\\"|', '', $tb_body);
             }
             return true;
         } else {
             return false;
         }
     } else {
         return false;
     }
 }

 /**
  * This method tries its best to convert the input string to UTF-8.
  *
  * Currently only ISO-5991-1 input and UTF-8 input is supported, but this
  * may be expanded upon if we receive other examples.
  *
  * @param string $str
  * @return string
  */
 public static function convertToUTF8($str)
 {
     $encoding = mb_detect_encoding($str, array('UTF-8', 'ISO-8859-1', 'WINDOWS-1252'), true);
     switch ($encoding) {
         case 'ISO-8859-1':
             $newStr = utf8_encode($str);
             break;
             /* Unreachable code. Not sure yet how we can improve this
                 * situation.
                case 'WINDOWS-1252' :
                    $newStr = iconv('cp1252', 'UTF-8', $str);
                    break;
                 */
         /* Unreachable code. Not sure yet how we can improve this
             * situation.
            case 'WINDOWS-1252' :
                $newStr = iconv('cp1252', 'UTF-8', $str);
                break;
             */
         default:
             $newStr = $str;
     }
     // Removing any control characters
     return preg_replace('%(?:[\\x00-\\x08\\x0B-\\x0C\\x0E-\\x1F\\x7F])%', '', $newStr);
 }

function getAuthResult($postData, $cookie_file)
{
    $url = 'http://rkk.cdpf.org.cn/queryDistrictrecordCDPF.action';
    $header = array("Host: rkk.cdpf.org.cn", 'Cache-Control: max-age=0', 'Connection: keep-alive', "Content-Length: " . strlen($postData), 'Origin: http://rkk.cdpf.org.cn', 'User-Agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36', 'Content-Type: application/x-www-form-urlencoded', 'Referer: http://rkk.cdpf.org.cn/content2.html', 'Accept-Encoding: gzip,deflate', 'Accept-Language: zh-CN,zh;q=0.8');
    $ch = curl_init();
    $res = curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, FALSE);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
    curl_setopt($ch, CURLOPT_HEADER, 0);
    curl_setopt($ch, CURLOPT_POST, 1);
    curl_setopt($ch, CURLOPT_POSTFIELDS, $postData);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
    curl_setopt($ch, CURLOPT_COOKIEFILE, $cookie_file);
    $result = curl_exec($ch);
    //需要转码
    echo '<br>';
    echo mb_detect_encoding($result, array('ASCII', 'UTF-8', 'GB2312', 'GBK', 'BIG5'));
    $result = mb_convert_encoding($result, 'UTF-8', 'EUC-CN');
    echo $result;
    $pos = strpos($result, '号码一致');
    if ($pos) {
        echo '号码一致';
    } else {
        echo '不一致';
    }
    curl_close($ch);
}

 static function GeoCode($address, $returnBox = false)
 {
     $params =& JComponentHelper::getParams('com_webmapplus');
     $key = $params->get('gmaps_api_key', '');
     $request = "http://maps.google.com/maps/geo?q=" . urlencode($address) . "&key={$key}";
     if (function_exists("curl_version")) {
         $ch = curl_init();
         curl_setopt($ch, CURLOPT_URL, $request);
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
         curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 0);
         $page = curl_exec($ch);
         curl_close($ch);
     } elseif (ini_get('allow_url_fopen') == 1) {
         $page = file_get_contents($request);
     } else {
         echo "cURL is not installed and allow_url_fopen is false. Can not continue.";
         die;
         return false;
     }
     //Silly Google doesn't use UTF-8 Encoding
     $page = mb_convert_encoding($page, 'UTF-8', mb_detect_encoding($page, 'UTF-8, ISO-8859-1', true));
     $data = json_decode($page);
     if ($data->Status->code == "200") {
         if (!$returnBox) {
             return $data->Placemark[0]->Point->coordinates;
         } else {
             return $data->Placemark[0]->ExtendedData->LatLonBox;
         }
     } else {
         return $data->Status->code;
     }
 }

 public function command_calc($user, $channel, $args)
 {
     $calc = implode(" ", $args);
     $data = file_get_contents("http://www.google.com/ig/calculator?hl=en&q=" . urlencode($calc));
     $this->bot->privmsg($channel, "0|" . $data);
     $data = mb_convert_encoding($data, 'UTF-8', mb_detect_encoding($data, 'UTF-8, ISO-8859-1', true));
     //print("1|".$data . "\n");
     $this->bot->privmsg($channel, "1|" . $data);
     $data = preg_replace("/([,{])(.*?):/", '$1"$2":', $data);
     //hack, convert js to json.
     //print("2|".$data . "\n");
     $this->bot->privmsg($channel, "2|" . $data);
     $data = stripcslashes($data);
     //print("3|".$data . "\n");
     $this->bot->privmsg($channel, "3|" . $data);
     $data = preg_replace("/<sup>(.*?)<\\/sup>/", '^$1', $data);
     //print("4|".$data . "\n");
     $this->bot->privmsg($channel, "4|" . $data);
     $data = preg_replace("/&#215;/", '\\u00d7', $data);
     //print("5|".$data . "\n");
     $this->bot->privmsg($channel, "5|" . $data);
     $data = json_decode($data, 1);
     //foreach ($data as &$node) { $node = html_entity_decode($node); }
     //print_r($data);
     $this->bot->privmsg($channel, "6|" . json_encode($data));
     $this->bot->privmsg($channel, $data["lhs"] . " = " . $data["rhs"]);
     //$this->bot->privmsg($channel, json_encode(array($data[error], $data[icc], $calc))); //DEBUG
 }

 public function encodeMessage($message)
 {
     if (mb_detect_encoding($message['message']) != $this->_encoding) {
         $message['message'] = mb_convert_encoding($message['message'], $this->_encoding);
     }
     return $message;
 }

function normalize($text, $separator = "-")
{
    $isUTF8 = mb_detect_encoding($text . " ", 'UTF-8,ISO-8859-1') == 'UTF-8';
    $text = $isUTF8 ? utf8_decode($text) : $text;
    $text = trim($text);
    $_a = utf8_decode("ÁÀãâàá");
    $_e = utf8_decode("ÉÈéè");
    $_i = utf8_decode("ÍÌíì");
    $_o = utf8_decode("ÓÒóò");
    $_u = utf8_decode("ÚÙúù");
    $_n = utf8_decode("Ññ");
    $_c = utf8_decode("Çç");
    $_b = utf8_decode("ß");
    $_dash = "\\.,_ ";
    $text = preg_replace("/[{$_a}]/", "a", $text);
    $text = preg_replace("/[{$_e}]/", "e", $text);
    $text = preg_replace("/[{$_i}]/", "i", $text);
    $text = preg_replace("/[{$_o}]/", "o", $text);
    $text = preg_replace("/[{$_u}]/", "u", $text);
    $text = preg_replace("/[{$_n}]/", "n", $text);
    $text = preg_replace("/[{$_c}]/", "c", $text);
    $text = preg_replace("/[{$_b}]/", "ss", $text);
    $text = preg_replace("/[{$_dash}]/", $separator, $text);
    $text = preg_replace("/[^a-zA-Z0-9\\-]/", "", $text);
    $text = strtolower($text);
    return $isUTF8 ? utf8_encode($text) : $text;
}

 /**
  * Converts content of the given file to UTF-8.
  *
  * @param string $filePath File path
  * @return string
  */
 public function convertFile($filePath)
 {
     $content = file_get_contents($filePath);
     static $cache = array();
     if (!isset($cache[$filePath])) {
         if (1 === count($this->charsets)) {
             // One character set
             $charset = $this->charsets[0];
         } else {
             // Detection
             $charset = mb_detect_encoding($content, $this->charsets);
             // The previous function can not handle WINDOWS-1250 and returns ISO-8859-2 instead
             if ('ISO-8859-2' === $charset && preg_match('~[\\x7F-\\x9F\\xBC]~', $content)) {
                 $charset = 'WINDOWS-1250';
             }
         }
         $cache[$filePath] = $charset;
     } else {
         $charset = $cache[$filePath];
     }
     if ('UTF-8' === $charset) {
         return $content;
     }
     return @iconv($charset, 'UTF-8//TRANSLIT//IGNORE', $content);
 }

function dump_dir_r($folder)
{
    if ($handle = opendir($folder)) {
        echo "------------------- {$folder} --------------------- \n";
        echo "Directory handle: {$handle}\n";
        echo "Files:\n";
        /* Das ist der korrekte Weg, ein Verzeichnis zu durchlaufen. */
        while (false !== ($file = readdir($handle))) {
            if ($file != "." && $file != "..") {
                if (is_dir($file)) {
                    dump_dir_r($file);
                } else {
                    # Other option would be "auto" to freely discover
                    $found_encoding = mb_detect_encoding($file, "UTF-8, ISO-8859-1");
                    echo "{$file}\n";
                    echo "Encoding detected:  {$found_encoding}  \n";
                    hex_dump("{$file}");
                    if (!($found_encoding == 'UTF-8' || $found_encoding == 'ASCII')) {
                        $file_utf8 = mb_convert_encoding($file, "UTF-8");
                        echo "\nConverted to UTF-8: {$file_utf8} \n";
                        hex_dump($file_utf8);
                    }
                }
            }
        }
        echo "---------------------------------------------------- \n";
        closedir($handle);
    }
}

function change_map_title($conn, $data)
{
    $id = $data['id'];
    $title = $data['title'];
    if (mb_detect_encoding($data['title'] . " ", 'UTF-8,ISO-8859-1') == 'UTF-8') {
        $title = mb_convert_encoding($data['title'], 'ISO-8859-1', 'UTF-8');
    }
    ossim_valid($title, OSS_INPUT, 'illegal:' . _('Title'));
    ossim_valid($id, OSS_HEX, 'illegal:' . _('Map'));
    if (ossim_error()) {
        $info_error = "Error: " . ossim_get_error();
        ossim_clean_error();
        $return['error'] = TRUE;
        $return['msg'] = $info_error;
        return $return;
    }
    if (!is_map_editable($conn, $id)) {
        $return['error'] = TRUE;
        $return['msg'] = _("You do not have permission to edit this map");
        return $return;
    }
    $query = "UPDATE risk_maps SET name = ? WHERE map = UNHEX(?)";
    $params = array($title, $id);
    if ($conn->Execute($query, $params) === FALSE) {
        $return['error'] = TRUE;
        $return['msg'] = $conn->ErrorMsg() . '.';
        return $return;
    }
    $return['error'] = FALSE;
    $return['msg'] = _('Map name changed');
    return $return;
}

function utf_correct($str)
{
    if (mb_detect_encoding($str, 'UTF-8, ISO-8859-1') != 'UTF-8') {
        $str = utf8_encode($str);
    }
    return $str;
}

 /**
  * Конвертирует строку на английском в строку на русском в 
  * соответвии с windows раскладкой клавиатуры
  * 
  * @param string $strMessage
  * @return string
  */
 public static function Key($strMessage)
 {
     //Если латинских символов нет, то раскладка верная — вернем строку, как она к нам пришла
     if (!preg_match('/[a-zA-Z]/u', $strMessage)) {
         return $strMessage;
     }
     $s1 = "qazwsxedcrfvtgbyhnujmik,ol.p;[']-1234567890 ";
     $s2 = "йфяцычувскамепинртгоьшлбщдюзжхэъ-1234567890 ";
     $s12 = "QAZWSXEDCRFVTGBYHNUJMIK<OL>P:{\"} ";
     $s22 = "ЙФЯЦЫЧУВСКАМЕПИНРТГОЬШЛБЩДЮЗЖХЭЪ ";
     $strNew = '';
     for ($i = 0; $i < mb_strlen($strMessage, mb_detect_encoding($strMessage)); $i++) {
         $char = mb_substr($strMessage, $i, 1, mb_detect_encoding($strMessage));
         if (strpos($s2, $char) !== false) {
             $strNew .= $char;
             continue;
         }
         if (strpos($s22, $char) !== false) {
             $strNew .= $char;
             continue;
         }
         if (strpos($s1, $char) !== false) {
             $p = strpos($s1, $char);
             $strNew .= mb_substr($s2, $p, 1, mb_detect_encoding($s2));
             continue;
         }
         if (strpos($s12, $char) !== false) {
             $p = strpos($s12, $char);
             $strNew .= mb_substr($s22, $p, 1, mb_detect_encoding($s22));
             continue;
         }
     }
     return $strNew;
 }

 /**
  * {@inheritdoc}
  */
 public function apply($instance, stdClass $schema, Context $context, Walker $walker)
 {
     $length = extension_loaded('mbstring') ? mb_strlen($instance, mb_detect_encoding($instance)) : strlen($instance);
     if ($length > $schema->maxLength) {
         $context->addViolation('should be lesser than or equal to %s characters', [$schema->maxLength]);
     }
 }

 /**
  * {@inheritdoc}
  */
 public function startQuery($sql, array $params = null, array $types = null)
 {
     if (null !== $this->stopwatch) {
         $this->stopwatch->start('doctrine', 'doctrine');
     }
     if (is_array($params)) {
         array_walk($params, function (&$param) {
             if (!is_string($param)) {
                 return;
             }
             // non utf-8 strings break json encoding
             if (!preg_match('#[\\p{L}\\p{N} ]#u', $param)) {
                 $param = self::BINARY_DATA_VALUE;
                 return;
             }
             // detect if the too long string must be shorten
             if (function_exists('mb_detect_encoding') && false !== ($encoding = mb_detect_encoding($param))) {
                 if (self::MAX_STRING_LENGTH < mb_strlen($param, $encoding)) {
                     $param = mb_substr($param, 0, self::MAX_STRING_LENGTH - 6, $encoding) . ' [...]';
                     return;
                 }
             } else {
                 if (self::MAX_STRING_LENGTH < strlen($param)) {
                     $param = substr($param, 0, self::MAX_STRING_LENGTH - 6) . ' [...]';
                     return;
                 }
             }
         });
     }
     if (null !== $this->logger) {
         $this->log($sql, null === $params ? array() : $params);
     }
 }

 protected function validateIdentical($input)
 {
     if (is_array($input)) {
         return end($input) === $this->endValue;
     }
     return mb_strrpos($input, $this->endValue, 0, $enc = mb_detect_encoding($input)) === mb_strlen($input, $enc) - mb_strlen($this->endValue, $enc);
 }

 function getSuggestion($word)
 {
     if (function_exists("mb_convert_encoding")) {
         $word = mb_convert_encoding($word, "ISO-8859-1", mb_detect_encoding($word, "UTF-8"));
     } else {
         $word = utf8_encode($word);
     }
     if ($fh = fopen($this->tmpfile, "w")) {
         fwrite($fh, "!\n");
         fwrite($fh, "^{$word}\n");
         fclose($fh);
     } else {
         die("Error opening tmp file.");
     }
     $data = shell_exec($this->cmd);
     @unlink($this->tmpfile);
     $returnData = array();
     $dataArr = preg_split("/\n/", $data, -1, PREG_SPLIT_NO_EMPTY);
     foreach ($dataArr as $dstr) {
         $matches = array();
         // Skip this line.
         if (strpos($dstr, "@") === 0) {
             continue;
         }
         preg_match("/\\& .* .* .*: (.*)/i", $dstr, $matches);
         if (!empty($matches[1])) {
             // For some reason, the exec version seems to add commas?
             $returnData[] = str_replace(",", "", $matches[1]);
         }
     }
     return $returnData;
 }

 /**
  * Sets lexer input.
  *
  * @param string $input    Input string
  * @param string $language Language name
  *
  * @throws Exception\LexerException
  */
 public function analyse($input, $language = 'en')
 {
     // try to detect unsupported encoding
     if ('UTF-8' !== mb_detect_encoding($input, 'UTF-8', true)) {
         throw new LexerException('Feature file is not in UTF8 encoding');
     }
     $input = strtr($input, array("\r\n" => "\n", "\r" => "\n"));
     $this->lines = explode("\n", $input);
     $this->linesCount = count($this->lines);
     $this->line = $this->lines[0];
     $this->lineNumber = 1;
     $this->trimmedLine = null;
     $this->eos = false;
     $this->deferredObjects = array();
     $this->deferredObjectsCount = 0;
     $this->stashedToken = null;
     $this->inPyString = false;
     $this->pyStringSwallow = 0;
     $this->featureStarted = false;
     $this->allowMultilineArguments = false;
     $this->allowSteps = false;
     $this->keywords->setLanguage($this->language = $language);
     $this->keywordsCache = array();
     $this->stepKeywordTypesCache = array();
 }

 public static function decode($data, $charset, &$o, &$attachments, $options = array())
 {
     if ($charset == 'unknown') {
         $charset = mb_detect_encoding($data);
     }
     return "<pre>" . iconv(strtoupper($charset), 'UTF-8', $data) . "</pre>";
 }

 public function testTokenize()
 {
     $tokens = self::$textApi->getTokensForText(self::$inputText, null, TestConfiguration::$RETINA_NAME);
     $this->assertNotEquals(count($tokens), 0);
     $this->assertTrue(mb_detect_encoding($tokens[0], 'UTF-8') == 'UTF-8');
     $this->assertEquals(explode(',', $tokens[0])[0], "george");
 }