function processPage($row)
 {
     global $wgContLang;
     $source = $row->img_name;
     if ($source == '') {
         // Ye olde empty rows. Just kill them.
         $this->killRow($source);
         return $this->progress(1);
     }
     $cleaned = $source;
     // About half of old bad image names have percent-codes
     $cleaned = rawurldecode($cleaned);
     // Some are old latin-1
     $cleaned = $wgContLang->checkTitleEncoding($cleaned);
     // Many of remainder look like non-normalized unicode
     $cleaned = UtfNormal::cleanUp($cleaned);
     $title = Title::makeTitleSafe(NS_IMAGE, $cleaned);
     if (is_null($title)) {
         $this->log("page {$source} ({$cleaned}) is illegal.");
         $safe = $this->buildSafeTitle($cleaned);
         $this->pokeFile($source, $safe);
         return $this->progress(1);
     }
     if ($title->getDbKey() !== $source) {
         $munged = $title->getDbKey();
         $this->log("page {$source} ({$munged}) doesn't match self.");
         $this->pokeFile($source, $munged);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 /**
  * Format an XML element as with self::element(), but run text through the
  * UtfNormal::cleanUp() validator first to ensure that no invalid UTF-8
  * is passed.
  *
  * @param $element String:
  * @param $attribs Array: Name=>value pairs. Values will be escaped.
  * @param $contents String: NULL to make an open tag only; '' for a contentless closed tag (default)
  * @return string
  */
 public static function elementClean($element, $attribs = array(), $contents = '')
 {
     if ($attribs) {
         $attribs = array_map(array('UtfNormal', 'cleanUp'), $attribs);
     }
     if ($contents) {
         $contents = UtfNormal::cleanUp($contents);
     }
     return self::element($element, $attribs, $contents);
 }

 /**
  * Format an XML element as with self::element(), but run text through the
  * UtfNormal::cleanUp() validator first to ensure that no invalid UTF-8
  * is passed.
  *
  * @param $element String:
  * @param $attribs Array: Name=>value pairs. Values will be escaped.
  * @param $contents String: NULL to make an open tag only; '' for a contentless closed tag (default)
  * @return string
  */
 public static function elementClean($element, $attribs = array(), $contents = '')
 {
     if ($attribs) {
         $attribs = array_map(array('UtfNormal', 'cleanUp'), $attribs);
     }
     if ($contents) {
         wfProfileIn(__METHOD__ . '-norm');
         $contents = UtfNormal::cleanUp($contents);
         wfProfileOut(__METHOD__ . '-norm');
     }
     return self::element($element, $attribs, $contents);
 }

function gs_utf8_decompose_to_ascii($str)
{
    static $map = null;
    if (!is_array($map)) {
        $map = _gs_utf8_get_map();
    }
    $str = UtfNormal::toNFD(strTr($str, $map));
    # return "safe" ASCII without control chars, newlines etc.
    //$str = preg_replace('/[^a-z0-9\-_. *#\'"!$()\/]/i', '', $str);
    $str = preg_replace('/[^\\x20-\\x7E]/', '', $str);
    return $str;
}

 function processPage($row)
 {
     $current = Title::makeTitle($row->wl_namespace, $row->wl_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if ($row->wl_user == 0 || is_null($title) || !$title->equals($current)) {
         $this->log("invalid watch by {$row->wl_user} for ({$row->wl_namespace}, \"{$row->wl_title}\")");
         $this->removeWatch($row);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 public static function formatDiffRow($title, $oldid, $newid, $timestamp, $comment, $actiontext = '')
 {
     global $wgFeedDiffCutoff, $wgContLang, $wgUser;
     wfProfileIn(__FUNCTION__);
     $skin = $wgUser->getSkin();
     # log enties
     $completeText = '<p>' . implode(' ', array_filter(array($actiontext, $skin->formatComment($comment)))) . "</p>\n";
     //NOTE: Check permissions for anonymous users, not current user.
     //      No "privileged" version should end up in the cache.
     //      Most feed readers will not log in anway.
     $anon = new User();
     $accErrors = $title->getUserPermissionsErrors('read', $anon, true);
     if ($title->getNamespace() >= 0 && !$accErrors) {
         if ($oldid) {
             wfProfileIn(__FUNCTION__ . "-dodiff");
             #$diffText = $de->getDiff( wfMsg( 'revisionasof',
             #	$wgContLang->timeanddate( $timestamp ) ),
             #	wfMsg( 'currentrev' ) );
             // Don't bother generating the diff if we won't be able to show it
             if ($wgFeedDiffCutoff > 0) {
                 $de = new DifferenceEngine($title, $oldid, $newid);
                 $diffText = $de->getDiff(wfMsg('previousrevision'), wfMsg('revisionasof', $wgContLang->timeanddate($timestamp)));
             }
             if (strlen($diffText) > $wgFeedDiffCutoff || $wgFeedDiffCutoff <= 0) {
                 // Omit large diffs
                 $diffLink = $title->escapeFullUrl('diff=' . $newid . '&oldid=' . $oldid);
                 $diffText = '<a href="' . $diffLink . '">' . htmlspecialchars(wfMsgForContent('showdiff')) . '</a>';
             } elseif ($diffText === false) {
                 // Error in diff engine, probably a missing revision
                 $diffText = "<p>Can't load revision {$newid}</p>";
             } else {
                 // Diff output fine, clean up any illegal UTF-8
                 $diffText = UtfNormal::cleanUp($diffText);
                 $diffText = self::applyDiffStyle($diffText);
             }
             wfProfileOut(__FUNCTION__ . "-dodiff");
         } else {
             $rev = Revision::newFromId($newid);
             if (is_null($rev)) {
                 $newtext = '';
             } else {
                 $newtext = $rev->getText();
             }
             $diffText = '<p><b>' . wfMsg('newpage') . '</b></p>' . '<div>' . nl2br(htmlspecialchars($newtext)) . '</div>';
         }
         $completeText .= $diffText;
     }
     wfProfileOut(__FUNCTION__);
     return $completeText;
 }

 /**
  * Returns the normalized form of the given page title, using the normalization rules of the given site.
  * If the given title is a redirect, the redirect weill be resolved and the redirect target is returned.
  *
  * @note  : This actually makes an API request to the remote site, so beware that this function is slow and depends
  *          on an external service.
  *
  * @note  : If MW_PHPUNIT_TEST is defined, the call to the external site is skipped, and the title
  *          is normalized using the local normalization rules as implemented by the Title class.
  *
  * @see Site::normalizePageName
  *
  * @since 1.21
  *
  * @param string $pageName
  *
  * @return string
  * @throws MWException
  */
 public function normalizePageName($pageName)
 {
     // Check if we have strings as arguments.
     if (!is_string($pageName)) {
         throw new MWException('$pageName must be a string');
     }
     // Go on call the external site
     if (defined('MW_PHPUNIT_TEST')) {
         // If the code is under test, don't call out to other sites, just normalize locally.
         // Note: this may cause results to be inconsistent with the actual normalization used by the respective remote site!
         $t = Title::newFromText($pageName);
         return $t->getPrefixedText();
     } else {
         // Make sure the string is normalized into NFC (due to the bug 40017)
         // but do nothing to the whitespaces, that should work appropriately.
         // @see https://bugzilla.wikimedia.org/show_bug.cgi?id=40017
         $pageName = UtfNormal::cleanUp($pageName);
         // Build the args for the specific call
         $args = array('action' => 'query', 'prop' => 'info', 'redirects' => true, 'converttitles' => true, 'format' => 'json', 'titles' => $pageName);
         $url = $this->getFileUrl('api.php') . '?' . wfArrayToCgi($args);
         // Go on call the external site
         //@todo: we need a good way to specify a timeout here.
         $ret = Http::get($url);
     }
     if ($ret === false) {
         wfDebugLog("MediaWikiSite", "call to external site failed: {$url}");
         return false;
     }
     $data = FormatJson::decode($ret, true);
     if (!is_array($data)) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned bad json: " . $ret);
         return false;
     }
     $page = static::extractPageRecord($data, $pageName);
     if (isset($page['missing'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned a marker for a missing page title! " . $ret);
         return false;
     }
     if (isset($page['invalid'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> returned a marker for an invalid page title! " . $ret);
         return false;
     }
     if (!isset($page['title'])) {
         wfDebugLog("MediaWikiSite", "call to <{$url}> did not return a page title! " . $ret);
         return false;
     }
     return $page['title'];
 }

 function processPage($row)
 {
     $current = Title::makeTitle($row->page_namespace, $row->page_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if (is_null($title)) {
         $this->log("page {$row->page_id} ({$display}) is illegal.");
         $this->moveIllegalPage($row);
         return $this->progress(1);
     }
     if (!$title->equals($current)) {
         $this->log("page {$row->page_id} ({$display}) doesn't match self.");
         $this->moveInconsistentPage($row, $title);
         return $this->progress(1);
     }
     $this->progress(0);
 }

 function makeInputText($max = false)
 {
     if ($max === false) {
         $max = $this->maxLength;
     }
     $length = mt_rand($this->minLength, $max);
     $s = '';
     for ($i = 0; $i < $length; $i++) {
         $hairIndex = mt_rand(0, count($this->hairs) - 1);
         $s .= $this->hairs[$hairIndex];
     }
     // Send through the UTF-8 normaliser
     // This resolves a few differences between the old preprocessor and the
     // XML-based one, which doesn't like illegals and converts line endings.
     // It's done by the MW UI, so it's a reasonably legitimate thing to do.
     $s = UtfNormal::cleanUp($s);
     return $s;
 }

 function processPage($row)
 {
     $current = Title::makeTitle($row->page_namespace, $row->page_title);
     $display = $current->getPrefixedText();
     $verified = UtfNormal::cleanUp($display);
     $title = Title::newFromText($verified);
     if (!is_null($title) && $title->equals($current) && $title->canExist()) {
         return $this->progress(0);
         // all is fine
     }
     if ($row->page_namespace == NS_FILE && $this->fileExists($row->page_title)) {
         $this->log("file {$row->page_title} needs cleanup, please run cleanupImages.php.");
         return $this->progress(0);
     } elseif (is_null($title)) {
         $this->log("page {$row->page_id} ({$display}) is illegal.");
         $this->moveIllegalPage($row);
         return $this->progress(1);
     } else {
         $this->log("page {$row->page_id} ({$display}) doesn't match self.");
         $this->moveInconsistentPage($row, $title);
         return $this->progress(1);
     }
 }

 /**
  * Function executed by use of {{#infoboxbuilder:}} parser function.
  * It gets the code from InfoboxBuilder.lua and creates new module object
  * from it. The module is then invoked and the result is returned.
  * @param  Parser  $parser Parser object
  * @param  PPFrame $frame  PPFrame object
  * @param  array   $args   Array of arguments passed from $frame object
  * @return string          A string returned by InfoboxBuilder.lua
  */
 public static function parserFunctionHook(\Parser $parser, $frame, $args)
 {
     wfProfileIn(__METHOD__);
     try {
         /**
          * Add the registered SCSS with the default theme
          */
         $parser->getOutput()->addModuleStyles('ext.wikia.InfoboxBuilder');
         $engine = \Scribunto::getParserEngine($parser);
         unset($args[0]);
         $childFrame = $frame->newChild($args, $parser->getTitle(), 1);
         $moduleText = file_get_contents(__DIR__ . '/includes/lua/InfoboxBuilder.lua');
         $module = new \Scribunto_LuaModule($engine, $moduleText, 'InfoboxBuilder');
         $result = $module->invoke('builder', $childFrame);
         $result = \UtfNormal::cleanUp(strval($result));
         wfProfileOut(__METHOD__);
         return $result;
     } catch (\ScribuntoException $e) {
         $trace = $e->getScriptTraceHtml(array('msgOptions' => array('content')));
         $html = \Html::element('p', array(), $e->getMessage());
         if ($trace !== false) {
             $html .= \Html::element('p', array(), wfMessage('scribunto-common-backtrace')->inContentLanguage()->text()) . $trace;
         }
         $out = $parser->getOutput();
         if (!isset($out->scribunto_errors)) {
             $out->addOutputHook('ScribuntoError');
             $out->scribunto_errors = array();
             $parser->addTrackingCategory('scribunto-common-error-category');
         }
         $out->scribunto_errors[] = $html;
         $id = 'mw-scribunto-error-' . (count($out->scribunto_errors) - 1);
         $parserError = wfMessage('scribunto-parser-error')->inContentLanguage()->text() . $parser->insertStripItem('<!--' . htmlspecialchars($e->getMessage()) . '-->');
         wfProfileOut(__METHOD__);
         // #iferror-compatible error element
         return "<strong class=\"error\"><span class=\"scribunto-error\" id=\"{$id}\">" . $parserError . "</span></strong>";
     }
 }

 /**
  * Get Normalized metadata in PHP-serialized form
  *
  * @param stdClass $video
  * @return string
  */
 protected static function getNormalizedMetadata($video)
 {
     // image.img_metadata
     $metadata = unserialize($video->img_metadata);
     foreach (self::$metadataFieldsContainingName as $field) {
         if (isset($metadata[$field])) {
             $metadata[$field] = \UtfNormal::toNFC($metadata[$field]);
         }
     }
     return serialize($metadata);
 }

 /**
  * Convert a UTF-8 string to normal form C. In Malayalam and Arabic, this
  * also cleans up certain backwards-compatible sequences, converting them
  * to the modern Unicode equivalent.
  *
  * This is language-specific for performance reasons only.
  *
  * @param $s string
  *
  * @return string
  */
 function normalize($s)
 {
     global $wgAllUnicodeFixes;
     $s = UtfNormal::cleanUp($s);
     if ($wgAllUnicodeFixes) {
         $s = $this->transformUsingPairFile('normalize-ar.ser', $s);
         $s = $this->transformUsingPairFile('normalize-ml.ser', $s);
     }
     return $s;
 }

function testLine($test, $line, &$total, &$success, &$failed, $columns, $exceptions, $verbose)
{
    $stripped = $line;
    UtfNormal::quickisNFCVerify($stripped);
    $same = $line == $stripped;
    $len = mb_strlen(substr($stripped, 0, strpos($stripped, '|')));
    if ($len == 0) {
        $len = strlen(substr($stripped, 0, strpos($stripped, '|')));
    }
    $ok = $same ^ $test >= 3;
    $ok ^= in_array($test, $exceptions);
    $ok &= $columns == $len;
    $total++;
    if ($ok) {
        $success++;
    } else {
        $failed++;
    }
    if ($verbose || !$ok) {
        print str_replace("\n", "{$len}\n", $stripped);
    }
}

 /**
  * Callback function for cleanUpUTF8()
  */
 private static function cleanUp_helper(&$s)
 {
     if (!is_string($s)) {
         return;
     }
     $s = UtfNormal::cleanUp($s);
 }

 /**
  * Converts a path consisting of object titles into a path consisting of tree
  * nodes. The comparison is non-case sensitive.
  *
  * Note: this function returns the same result as getNodePath, 
  * but takes a title path as parameter.
  *
  * @access	public
  * @param	Array	Path array with object titles.
  *                       e.g. array('ILIAS','English','Course A')
  * @param	ref_id	Startnode of the relative path. 
  *                       Specify null, if the title path is an absolute path.
  *                       Specify a ref id, if the title path is a relative 
  *                       path starting at this ref id.
  * @return	array	ordered path info (depth,parent,child,obj_id,type,title)
  *               or null, if the title path can not be converted into a node path.
  */
 function getNodePathForTitlePath($titlePath, $a_startnode_id = null)
 {
     global $ilDB, $log;
     //$log->write('getNodePathForTitlePath('.implode('/',$titlePath));
     // handle empty title path
     if ($titlePath == null || count($titlePath) == 0) {
         if ($a_startnode_id == 0) {
             return null;
         } else {
             return $this->getNodePath($a_startnode_id);
         }
     }
     // fetch the node path up to the startnode
     if ($a_startnode_id != null && $a_startnode_id != 0) {
         // Start using the node path to the root of the relative path
         $nodePath = $this->getNodePath($a_startnode_id);
         $parent = $a_startnode_id;
     } else {
         // Start using the root of the tree
         $nodePath = array();
         $parent = 0;
     }
     // Convert title path into Unicode Normal Form C
     // This is needed to ensure that we can compare title path strings with
     // strings from the database.
     require_once 'include/Unicode/UtfNormal.php';
     include_once './Services/Utilities/classes/class.ilStr.php';
     $inClause = 'd.title IN (';
     for ($i = 0; $i < count($titlePath); $i++) {
         $titlePath[$i] = ilStr::strToLower(UtfNormal::toNFC($titlePath[$i]));
         if ($i > 0) {
             $inClause .= ',';
         }
         $inClause .= $ilDB->quote($titlePath[$i], 'text');
     }
     $inClause .= ')';
     // Fetch all rows that are potential path elements
     if ($this->table_obj_reference) {
         $joinClause = 'JOIN ' . $this->table_obj_reference . '  r ON t.child = r.' . $this->ref_pk . ' ' . 'JOIN ' . $this->table_obj_data . ' d ON r.' . $this->obj_pk . ' = d.' . $this->obj_pk;
     } else {
         $joinClause = 'JOIN ' . $this->table_obj_data . '  d ON t.child = d.' . $this->obj_pk;
     }
     // The ORDER BY clause in the following SQL statement ensures that,
     // in case of a multiple objects with the same title, always the Object
     // with the oldest ref_id is chosen.
     // This ensure, that, if a new object with the same title is added,
     // WebDAV clients can still work with the older object.
     $q = 'SELECT t.depth, t.parent, t.child, d.' . $this->obj_pk . ' obj_id, d.type, d.title ' . 'FROM ' . $this->table_tree . '  t ' . $joinClause . ' ' . 'WHERE ' . $inClause . ' ' . 'AND t.depth <= ' . (count($titlePath) + count($nodePath)) . ' ' . 'AND t.tree = 1 ' . 'ORDER BY t.depth, t.child ASC';
     $r = $ilDB->query($q);
     $rows = array();
     while ($row = $r->fetchRow(DB_FETCHMODE_ASSOC)) {
         $row['title'] = UtfNormal::toNFC($row['title']);
         $row['ref_id'] = $row['child'];
         $rows[] = $row;
     }
     // Extract the path elements from the fetched rows
     for ($i = 0; $i < count($titlePath); $i++) {
         $pathElementFound = false;
         foreach ($rows as $row) {
             if ($row['parent'] == $parent && ilStr::strToLower($row['title']) == $titlePath[$i]) {
                 // FIXME - We should test here, if the user has
                 // 'visible' permission for the object.
                 $nodePath[] = $row;
                 $parent = $row['child'];
                 $pathElementFound = true;
                 break;
             }
         }
         // Abort if we haven't found a path element for the current depth
         if (!$pathElementFound) {
             //$log->write('ilTree.getNodePathForTitlePath('.var_export($titlePath,true).','.$a_startnode_id.'):null');
             return null;
         }
     }
     // Return the node path
     //$log->write('ilTree.getNodePathForTitlePath('.var_export($titlePath,true).','.$a_startnode_id.'):'.var_export($nodePath,true));
     return $nodePath;
 }

 /**
  * Return the original filename of the uploaded file, as reported by
  * the submitting user agent. HTML-style character entities are
  * interpreted and normalized to Unicode normalization form C, in part
  * to deal with weird input from Safari with non-ASCII filenames.
  *
  * Other than this the name is not verified for being a safe filename.
  *
  * @param $key String: 
  * @return string or NULL if no such file.
  */
 function getFileName($key)
 {
     if (!isset($_FILES[$key])) {
         return NULL;
     }
     $name = $_FILES[$key]['name'];
     # Safari sends filenames in HTML-encoded Unicode form D...
     # Horrid and evil! Let's try to make some kind of sense of it.
     $name = Sanitizer::decodeCharReferences($name);
     $name = UtfNormal::cleanUp($name);
     wfDebug("WebRequest::getFileName() '" . $_FILES[$key]['name'] . "' normalized to '{$name}'\n");
     return $name;
 }

 /**
  * Recursively normalizes UTF-8 strings in the given array.
  *
  * @param $data string|array
  * @return array|string cleaned-up version of the given
  * @private
  */
 function normalizeUnicode($data)
 {
     if (is_array($data)) {
         foreach ($data as $key => $val) {
             $data[$key] = $this->normalizeUnicode($val);
         }
     } else {
         global $wgContLang;
         $data = isset($wgContLang) ? $wgContLang->normalize($data) : UtfNormal::cleanUp($data);
     }
     return $data;
 }

 /**
  * Get the normalized composed version of the title
  *
  * @return string
  */
 public function getNormalizedDestinationTitle()
 {
     return \UtfNormal::toNFC($this->getSanitizedTitleText());
 }

 /**
  * Helper function of a helper function to convert charset for iptc values.
  * @param string|array $data The IPTC string
  * @param string $charset The charset
  *
  * @return string
  */
 private static function convIPTCHelper($data, $charset)
 {
     if ($charset) {
         wfSuppressWarnings();
         $data = iconv($charset, "UTF-8//IGNORE", $data);
         wfRestoreWarnings();
         if ($data === false) {
             $data = "";
             wfDebugLog('iptc', __METHOD__ . " Error converting iptc data charset {$charset} to utf-8");
         }
     } else {
         //treat as utf-8 if is valid utf-8. otherwise pretend its windows-1252
         // most of the time if there is no 1:90 tag, it is either ascii, latin1, or utf-8
         $oldData = $data;
         UtfNormal::quickIsNFCVerify($data);
         //make $data valid utf-8
         if ($data === $oldData) {
             return $data;
             //if validation didn't change $data
         } else {
             return self::convIPTCHelper($oldData, 'Windows-1252');
         }
     }
     return trim($data);
 }