Operate on text using multibyte functions

TinyIB now fully supports UTF-8 encoded text input.

Resolves #255.
Resolves #273.
This commit is contained in:
Trevor Slocum 2023-09-24 14:17:24 -07:00
parent 0405fe9577
commit 5f42b82cc0
4 changed files with 52 additions and 17 deletions

View file

@ -81,6 +81,7 @@ support in mind.
- Set ``TINYIB_THUMBNAIL`` to ``imagemagick``.
- **Note:** GIF files will have animated thumbnails, which will often have large file sizes.
- To use TINYIB in another language, set ``TINYIB_LOCALE`` to a language code found in `locale/`.
- **Note:** The [mbstring](https://www.php.net/manual/en/book.mbstring.php) PHP extension must be installed and enabled for TinyIB to properly support operating on and rendering text in any language other than English.
6. [CHMOD](https://en.wikipedia.org/wiki/Chmod) write permissions to these directories:
- ./ (the directory containing TinyIB)
- ./src/

View file

@ -306,24 +306,24 @@ if (!isset($_GET['delete']) && !isset($_GET['manage']) && (isset($_POST['name'])
if ($staffpost || !in_array('name', $hide_fields)) {
list($post['name'], $post['tripcode']) = nameAndTripcode($_POST['name']);
$post['name'] = cleanString(substr($post['name'], 0, 75));
$post['name'] = cleanString(_substr($post['name'], 0, 75));
if (!$staffpost && TINYIB_MAXNAME > 0) {
$post['name'] = substr($post['name'], 0, TINYIB_MAXNAME);
$post['name'] = _substr($post['name'], 0, TINYIB_MAXNAME);
}
}
if ($staffpost || !in_array('email', $hide_fields)) {
$post['email'] = cleanString(str_replace('"', '"', substr($_POST['email'], 0, 75)));
$post['email'] = cleanString(str_replace('"', '"', _substr($_POST['email'], 0, 75)));
if (!$staffpost && TINYIB_MAXEMAIL > 0) {
$post['email'] = substr($post['email'], 0, TINYIB_MAXEMAIL);
$post['email'] = _substr($post['email'], 0, TINYIB_MAXEMAIL);
}
}
if ($staffpost) {
$capcode = ($isadmin) ? ' <span style="color: ' . $tinyib_capcodes[0][1] . ' ;">## ' . $tinyib_capcodes[0][0] . '</span>' : ' <span style="color: ' . $tinyib_capcodes[1][1] . ';">## ' . $tinyib_capcodes[1][0] . '</span>';
}
if ($staffpost || !in_array('subject', $hide_fields)) {
$post['subject'] = cleanString(substr($_POST['subject'], 0, 75));
$post['subject'] = cleanString(_substr($_POST['subject'], 0, 75));
if (!$staffpost && TINYIB_MAXSUBJECT > 0) {
$post['subject'] = substr($post['subject'], 0, TINYIB_MAXSUBJECT);
$post['subject'] = _substr($post['subject'], 0, TINYIB_MAXSUBJECT);
}
}
if ($staffpost || !in_array('message', $hide_fields)) {
@ -332,7 +332,7 @@ if (!isset($_GET['delete']) && !isset($_GET['manage']) && (isset($_POST['name'])
// Treat message as raw HTML
} else {
if (TINYIB_WORDBREAK > 0) {
$post['message'] = preg_replace('/([^\s]{' . TINYIB_WORDBREAK . '})(?=[^\s])/', '$1' . TINYIB_WORDBREAK_IDENTIFIER, $post['message']);
$post['message'] = preg_replace('/([^\s]{' . TINYIB_WORDBREAK . '})(?=[^\s])/u', '$1' . TINYIB_WORDBREAK_IDENTIFIER, $post['message']);
}
$post['message'] = str_replace("\n", '<br>', makeLinksClickable(colorQuote(postLink(cleanString(rtrim($post['message']))))));
@ -1066,8 +1066,8 @@ EOF;
$action = sprintf(__('Deleted %s'),'&gt;&gt;' . $post['id']) . ' - ' . hashData($post['ip']);
$stripped = strip_tags($post['message']);
if ($stripped != '') {
$action .= ' - ' . htmlentities(substr($stripped, 0, 32));
if (strlen($stripped) > 32) {
$action .= ' - ' . htmlentities(_substr($stripped, 0, 32));
if (_strlen($stripped) > 32) {
$action .= '...';
}
}

View file

@ -3,6 +3,8 @@ if (!defined('TINYIB_BOARD')) {
die('');
}
$multibyte_enabled = function_exists('mb_strlen');
if (!function_exists('array_column')) {
function array_column($array, $column_name) {
return array_map(function ($element) use ($column_name) {
@ -24,6 +26,38 @@ function lockDatabase() {
return $fp;
}
function _strlen($string) {
global $multibyte_enabled;
if ($multibyte_enabled) {
return mb_strlen($string);
}
return strlen($string);
}
function _strpos($haystack, $needle, $offset=0) {
global $multibyte_enabled;
if ($multibyte_enabled) {
return mb_strpos($haystack, $needle, $offset);
}
return strpos($haystack, $needle, $offset);
}
function _substr($string, $start, $length=null) {
global $multibyte_enabled;
if ($multibyte_enabled) {
return mb_substr($string, $start, $length);
}
return substr($string, $start, $length);
}
function _substr_count($haystack, $needle) {
global $multibyte_enabled;
if ($multibyte_enabled) {
return mb_substr_count($haystack, $needle);
}
return substr_count($haystack, $needle);
}
function hashData($data, $force = false) {
global $bcrypt_salt;
if (substr($data, 0, 4) == '$2y$' && !$force) {
@ -388,8 +422,8 @@ function checkFlood() {
}
function checkMessageSize() {
if (TINYIB_MAXMESSAGE > 0 && strlen($_POST['message']) > TINYIB_MAXMESSAGE) {
fancyDie(sprintf(__('Please shorten your message, or post it in multiple parts. Your message is %1$d characters long, and the maximum allowed is %2$d.'), strlen($_POST['message']), TINYIB_MAXMESSAGE));
if (TINYIB_MAXMESSAGE > 0 && _strlen($_POST['message']) > TINYIB_MAXMESSAGE) {
fancyDie(sprintf(__('Please shorten your message, or post it in multiple parts. Your message is %1$d characters long, and the maximum allowed is %2$d.'), _strlen($_POST['message']), TINYIB_MAXMESSAGE));
}
}
@ -793,8 +827,8 @@ function addVideoOverlay($thumb_location) {
function strallpos($haystack, $needle, $offset = 0) {
$result = array();
for ($i = $offset; $i < strlen($haystack); $i++) {
$pos = strpos($haystack, $needle, $i);
for ($i = $offset; $i < _strlen($haystack); $i++) {
$pos = _strpos($haystack, $needle, $i);
if ($pos !== False) {
$offset = $pos;
if ($offset >= $i) {
@ -900,7 +934,7 @@ function attachFile($post, $filepath, $filename, $uploaded, $spoiler) {
}
$post['file'] = $file_name;
$post['file_original'] = trim(htmlentities(substr($filename, 0, 50), ENT_QUOTES));
$post['file_original'] = trim(htmlentities(_substr($filename, 0, 50), ENT_QUOTES));
$post['file_hex'] = md5_file($filepath);
$post['file_size'] = $filesize;
$post['file_size_formatted'] = convertBytes($post['file_size']);

View file

@ -594,9 +594,9 @@ EOF;
$return .= backlinks($post);
}
if (TINYIB_TRUNCATE > 0 && !$res && substr_count($post['message'], '<br>') > TINYIB_TRUNCATE) { // Truncate messages on board index pages for readability
if (TINYIB_TRUNCATE > 0 && !$res && _substr_count($post['message'], '<br>') > TINYIB_TRUNCATE) { // Truncate messages on board index pages for readability
$br_offsets = strallpos($post['message'], '<br>');
$post['message'] = substr($post['message'], 0, $br_offsets[TINYIB_TRUNCATE - 1]);
$post['message'] = _substr($post['message'], 0, $br_offsets[TINYIB_TRUNCATE - 1]);
$post['message'] .= '<br><span class="omittedposts">' . __('Post truncated. Click Reply to view.') . '</span><br>';
}
$return .= <<<EOF
@ -762,7 +762,7 @@ function buildCatalogPost($post) {
EOF;
}
$replies = numRepliesToThreadByID($post['id']);
$subject = trim($post['subject']) != '' ? $post['subject'] : substr(trim(str_ireplace("\n", '', strip_tags($post['message']))), 0, 75);
$subject = trim($post['subject']) != '' ? $post['subject'] : _substr(trim(str_ireplace("\n", '', strip_tags($post['message']))), 0, 75);
return <<<EOF
<div class="catalogpost" style="max-width: {$maxwidth}px;">