From 3ada3b76f0ce86274647758db029d84a0f404886 Mon Sep 17 00:00:00 2001 From: David M Date: Tue, 14 Aug 2007 18:53:17 +0000 Subject: add a noramlizer wrapper just for updates git-svn-id: file:///svn/phpbb/trunk@8032 89ea8834-ac86-4346-8a33-228a782c2dd0 --- phpBB/install/data/confusables.php | 2 +- phpBB/install/data/new_normalizer.php | 181 ++++++++++++++++++++++++++++++++++ phpBB/install/database_update.php | 12 ++- 3 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 phpBB/install/data/new_normalizer.php diff --git a/phpBB/install/data/confusables.php b/phpBB/install/data/confusables.php index 86223bd726..4472946a55 100644 --- a/phpBB/install/data/confusables.php +++ b/phpBB/install/data/confusables.php @@ -621,7 +621,7 @@ function utf8_new_case_fold_nfkc($text, $option = 'full') } // convert to NFKC - utf_normalizer::nfkc($text); + utf_new_normalizer::nfkc($text); // FC_NFKC_Closure, http://www.unicode.org/Public/5.0.0/ucd/DerivedNormalizationProps.txt $text = strtr($text, $fc_nfkc_closure); diff --git a/phpBB/install/data/new_normalizer.php b/phpBB/install/data/new_normalizer.php new file mode 100644 index 0000000000..a050520690 --- /dev/null +++ b/phpBB/install/data/new_normalizer.php @@ -0,0 +1,181 @@ + $string) + { + if (is_array($string)) + { + foreach ($string as $_key => $_string) + { + utf_new_normalizer::nfc($strings[$key][$_key]); + } + } + else + { + utf_new_normalizer::nfc($strings[$key]); + } + } + } + + return $strings; +} + +class utf_new_normalizer +{ + /** + * Validate, cleanup and normalize a string + * + * The ultimate convenience function! Clean up invalid UTF-8 sequences, + * and convert to Normal Form C, canonical composition. + * + * @param string &$str The dirty string + * @return string The same string, all shiny and cleaned-up + */ + function cleanup(&$str) + { + // The string below is the list of all autorized characters, sorted by frequency in latin text + $pos = strspn($str, "\x20\x65\x69\x61\x73\x6E\x74\x72\x6F\x6C\x75\x64\x5D\x5B\x63\x6D\x70\x27\x0A\x67\x7C\x68\x76\x2E\x66\x62\x2C\x3A\x3D\x2D\x71\x31\x30\x43\x32\x2A\x79\x78\x29\x28\x4C\x39\x41\x53\x2F\x50\x22\x45\x6A\x4D\x49\x6B\x33\x3E\x35\x54\x3C\x44\x34\x7D\x42\x7B\x38\x46\x77\x52\x36\x37\x55\x47\x4E\x3B\x4A\x7A\x56\x23\x48\x4F\x57\x5F\x26\x21\x4B\x3F\x58\x51\x25\x59\x5C\x09\x5A\x2B\x7E\x5E\x24\x40\x60\x7F\x0D"); + $len = strlen($str); + + if ($pos == $len) + { + // ASCII strings with no special chars return immediately + return; + } + + // Note: we do not check for $GLOBALS['utf_canonical_decomp']. It is assumed they are always loaded together + if (!isset($GLOBALS['utf_nfc_qc'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx); + } + + if (!isset($GLOBALS['utf_canonical_decomp'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx); + } + + // Replace any byte in the range 0x00..0x1F, except for \r, \n and \t + // We replace those characters with a 0xFF byte, which is illegal in UTF-8 and will in turn be replaced with a UTF replacement char + $str = strtr( + $str, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F", + "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" + ); + + $str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']); + } + + /** + * Validate and normalize a UTF string to NFC + * + * @param string &$str Unchecked UTF string + * @return string The string, validated and in normal form + */ + function nfc(&$str) + { + $pos = strspn($str, UTF8_ASCII_RANGE); + $len = strlen($str); + + if ($pos == $len) + { + // ASCII strings return immediately + return; + } + + if (!isset($GLOBALS['utf_nfc_qc'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx); + } + + if (!isset($GLOBALS['utf_canonical_decomp'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx); + } + + $str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']); + } + + /** + * Validate and normalize a UTF string to NFKC + * + * @param string &$str Unchecked UTF string + * @return string The string, validated and in normal form + */ + function nfkc(&$str) + { + $pos = strspn($str, UTF8_ASCII_RANGE); + $len = strlen($str); + + if ($pos == $len) + { + // ASCII strings return immediately + return; + } + + if (!isset($GLOBALS['utf_nfkc_qc'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_nfkc_qc.' . $phpEx); + } + + if (!isset($GLOBALS['utf_compatibility_decomp'])) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_compatibility_decomp.' . $phpEx); + } + + $str = utf_new_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfkc_qc'], $GLOBALS['utf_compatibility_decomp']); + } + + /** + * Recompose a UTF string + * + * @param string $str Unchecked UTF string + * @param integer $pos Position of the first UTF char (in bytes) + * @param integer $len Length of the string (in bytes) + * @param array &$qc Quick-check array, passed by reference but never modified + * @param array &$decomp_map Decomposition mapping, passed by reference but never modified + * @return string The string, validated and recomposed + * + * @access private + */ + function recompose($str, $pos, $len, &$qc, &$decomp_map) + { + global $utf_canonical_comp; + + // Load the canonical composition table + if (!isset($utf_canonical_comp)) + { + global $phpbb_root_path, $phpEx; + include($phpbb_root_path . 'includes/utf/data/utf_canonical_comp.' . $phpEx); + } + + return utf_normalizer::recompose($str, $pos, $len, &$qc, &$decomp_map); + } +} + +?> \ No newline at end of file diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index da8f6e9518..59a6a21689 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -567,6 +567,16 @@ if (version_compare($current_version, '3.0.RC4', '<=')) $modify_users = request_var('modify_users', array(0 => '')); $new_usernames = request_var('new_usernames', array(0 => ''), true); + if (!class_exists('utf_new_normalizer')) + { + if (!file_exists($phpbb_root_path . 'install/data/new_normalizer.' . $phpEx)) + { + global $lang; + trigger_error(sprintf($lang['UPDATE_REQUIRES_FILE'], $phpbb_root_path . 'install/data/new_normalizer.' . $phpEx), E_USER_ERROR); + } + include($phpbb_root_path . 'install/data/new_normalizer.' . $phpEx); + } + // the admin decided to change some usernames if (sizeof($modify_users) && $submit) { @@ -601,7 +611,7 @@ if (version_compare($current_version, '3.0.RC4', '<=')) case 'edit': if (isset($new_usernames[$user_id])) { - $data = array('username' => utf8_normalize_nfc($new_usernames[$user_id])); + $data = array('username' => utf8_new_normalize_nfc($new_usernames[$user_id])); // Need to update config, forum, topic, posting, messages, etc. if ($data['username'] != $row['username']) { -- cgit v1.2.3-65-gdbad