Harden mojibake repair with Czech-specific serialized value handling
This commit is contained in:
+194
-82
@@ -1,8 +1,8 @@
|
||||
<?php
|
||||
/**
|
||||
* Plugin Name: Mojibake Repair (Temporary)
|
||||
* Description: Previews and repairs the proven UTF-8/Latin-1 mojibake problem in standard WordPress core text columns.
|
||||
* Version: 1.0.0
|
||||
* Description: Previews and repairs Czech UTF-8 mojibake while preserving serialized WordPress settings.
|
||||
* Version: 1.1.0
|
||||
* Requires PHP: 7.4
|
||||
*/
|
||||
|
||||
@@ -14,6 +14,8 @@ final class WP_Mojibake_Repair_Temporary
|
||||
const SAMPLE_LIMIT = 5;
|
||||
const SAMPLE_LENGTH = 300;
|
||||
const APPROVAL_TTL = 600;
|
||||
const REPAIR_VERSION = 2;
|
||||
const BATCH_SIZE = 200;
|
||||
|
||||
public static function init()
|
||||
{
|
||||
@@ -61,7 +63,7 @@ final class WP_Mojibake_Repair_Temporary
|
||||
isset($_POST['approval']) ? wp_unslash($_POST['approval']) : '',
|
||||
$plan
|
||||
);
|
||||
$result = self::execute($plan, $approval['counts']);
|
||||
$result = self::execute($plan, $approval);
|
||||
}
|
||||
} catch (Throwable $e) {
|
||||
$error = $e->getMessage();
|
||||
@@ -164,116 +166,217 @@ final class WP_Mojibake_Repair_Temporary
|
||||
}
|
||||
}
|
||||
|
||||
private static function preview($plan)
|
||||
private static function preview($plan, $write = false)
|
||||
{
|
||||
global $wpdb;
|
||||
$preview = array('columns' => array(), 'counts' => array(), 'total' => 0, 'serialized' => 0);
|
||||
$preview = array('columns' => array(), 'counts' => array(), 'fingerprints' => array(), 'total' => 0, 'skipped' => 0);
|
||||
|
||||
foreach ($plan as $table => $definition) {
|
||||
foreach ($definition['columns'] as $column) {
|
||||
$table_sql = self::identifier($table);
|
||||
$key_sql = self::identifier($definition['key']);
|
||||
$column_sql = self::identifier($column);
|
||||
$converted = self::conversion($column);
|
||||
$diff = self::difference($column);
|
||||
$id = $table . '.' . $column;
|
||||
|
||||
$count = (int) $wpdb->get_var("SELECT COUNT(*) FROM {$table_sql} WHERE {$diff}");
|
||||
self::throw_on_db_error();
|
||||
|
||||
$serialized = (int) $wpdb->get_var(
|
||||
"SELECT COUNT(*) FROM {$table_sql} WHERE {$diff} AND (LEFT({$column_sql},2) IN ('a:','O:','s:','i:','d:','b:','C:') OR {$column_sql}='N;')"
|
||||
);
|
||||
self::throw_on_db_error();
|
||||
|
||||
$count = 0;
|
||||
$skipped = 0;
|
||||
$samples = array();
|
||||
if ($count > 0) {
|
||||
$limit = self::SAMPLE_LIMIT;
|
||||
$length = self::SAMPLE_LENGTH;
|
||||
$samples = $wpdb->get_results(
|
||||
"SELECT {$key_sql} AS row_id, LEFT({$column_sql},{$length}) AS before_value, LEFT({$converted},{$length}) AS after_value " .
|
||||
"FROM {$table_sql} WHERE {$diff} ORDER BY {$key_sql} ASC LIMIT {$limit}",
|
||||
$skipped_ids = array();
|
||||
$cursor = null;
|
||||
$hash = hash_init('sha256');
|
||||
|
||||
do {
|
||||
$where = $cursor === null ? '' : $wpdb->prepare("WHERE {$key_sql} > %d", $cursor);
|
||||
$limit = self::BATCH_SIZE;
|
||||
$batch = $wpdb->get_results(
|
||||
"SELECT {$key_sql} AS row_id, {$column_sql} AS before_value FROM {$table_sql} {$where} ORDER BY {$key_sql} ASC LIMIT {$limit}",
|
||||
ARRAY_A
|
||||
);
|
||||
self::throw_on_db_error();
|
||||
}
|
||||
foreach ($batch as $row) {
|
||||
$cursor = $row['row_id'];
|
||||
if ($row['before_value'] === null) {
|
||||
continue;
|
||||
}
|
||||
$before = (string) $row['before_value'];
|
||||
try {
|
||||
$after = self::repair_value($before);
|
||||
} catch (RuntimeException $e) {
|
||||
$skipped++;
|
||||
if (count($skipped_ids) < self::SAMPLE_LIMIT) {
|
||||
$skipped_ids[] = (string) $cursor;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ($after === $before) {
|
||||
continue;
|
||||
}
|
||||
// Length framing makes the approval cover exact IDs and full values.
|
||||
foreach (array((string) $cursor, $before, $after) as $part) {
|
||||
hash_update($hash, strlen($part) . ':' . $part);
|
||||
}
|
||||
$count++;
|
||||
if (count($samples) < self::SAMPLE_LIMIT) {
|
||||
$samples[] = array('row_id' => $cursor, 'before_value' => self::sample($before), 'after_value' => self::sample($after));
|
||||
}
|
||||
if ($write) {
|
||||
// Bypass option/meta APIs to preserve JSON escaping and serialization.
|
||||
// Refuse to overwrite a value changed since it was read.
|
||||
$affected = $wpdb->query($wpdb->prepare(
|
||||
"UPDATE {$table_sql} SET {$column_sql} = %s WHERE {$key_sql} = %d AND BINARY {$column_sql} = BINARY %s",
|
||||
$after,
|
||||
$cursor,
|
||||
$before
|
||||
));
|
||||
self::throw_on_db_error();
|
||||
if ($affected !== 1) {
|
||||
throw new RuntimeException('A value changed while updating ' . $id . '. Generate a fresh preview.');
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (count($batch) === self::BATCH_SIZE);
|
||||
|
||||
$preview['columns'][] = array(
|
||||
'table' => $table,
|
||||
'column' => $column,
|
||||
'count' => $count,
|
||||
'serialized' => $serialized,
|
||||
'samples' => $samples,
|
||||
'table' => $table, 'column' => $column, 'count' => $count,
|
||||
'skipped' => $skipped, 'skipped_ids' => $skipped_ids, 'samples' => $samples,
|
||||
);
|
||||
$preview['counts'][$id] = $count;
|
||||
$preview['fingerprints'][$id] = hash_final($hash);
|
||||
$preview['total'] += $count;
|
||||
$preview['serialized'] += $serialized;
|
||||
$preview['skipped'] += $skipped;
|
||||
}
|
||||
}
|
||||
|
||||
return $preview;
|
||||
}
|
||||
|
||||
private static function execute($plan, $approved_counts)
|
||||
private static function sample($value)
|
||||
{
|
||||
// Avoid cutting a UTF-8 character in the preview (no mbstring required).
|
||||
preg_match('/\A.{0,' . self::SAMPLE_LENGTH . '}/us', $value, $match);
|
||||
return isset($match[0]) ? $match[0] : '';
|
||||
}
|
||||
|
||||
private static function execute($plan, $approval)
|
||||
{
|
||||
global $wpdb;
|
||||
|
||||
$current = self::preview($plan);
|
||||
if ($current['counts'] !== $approved_counts) {
|
||||
throw new RuntimeException('The candidate counts changed after preview. Generate a fresh preview.');
|
||||
}
|
||||
|
||||
if ($wpdb->query('START TRANSACTION') === false) {
|
||||
self::throw_on_db_error();
|
||||
throw new RuntimeException('Could not start the repair transaction.');
|
||||
}
|
||||
|
||||
$rows = array();
|
||||
$total = 0;
|
||||
try {
|
||||
foreach ($plan as $table => $definition) {
|
||||
foreach ($definition['columns'] as $column) {
|
||||
$id = $table . '.' . $column;
|
||||
$expected = isset($approved_counts[$id]) ? (int) $approved_counts[$id] : -1;
|
||||
$table_sql = self::identifier($table);
|
||||
$column_sql = self::identifier($column);
|
||||
$converted = self::conversion($column);
|
||||
$diff = self::difference($column);
|
||||
|
||||
$affected = $wpdb->query("UPDATE {$table_sql} SET {$column_sql}={$converted} WHERE {$diff}");
|
||||
if ($affected === false) {
|
||||
self::throw_on_db_error();
|
||||
}
|
||||
if ((int) $affected !== $expected) {
|
||||
throw new RuntimeException('Affected-row count changed while updating ' . $id . '.');
|
||||
}
|
||||
|
||||
$rows[] = array('table' => $table, 'column' => $column, 'affected' => (int) $affected);
|
||||
$total += (int) $affected;
|
||||
}
|
||||
$current = self::preview($plan, true);
|
||||
if ($current['counts'] !== $approval['counts'] || $current['fingerprints'] !== $approval['fingerprints']) {
|
||||
throw new RuntimeException('Candidate values changed after preview. All changes were rolled back. Generate a fresh preview.');
|
||||
}
|
||||
|
||||
if ($wpdb->query('COMMIT') === false) {
|
||||
self::throw_on_db_error();
|
||||
throw new RuntimeException('Could not commit the repair transaction.');
|
||||
}
|
||||
} catch (Throwable $e) {
|
||||
$wpdb->query('ROLLBACK');
|
||||
throw $e;
|
||||
}
|
||||
|
||||
return array('rows' => $rows, 'total' => $total);
|
||||
// SQL writes bypass WordPress cache invalidation, including alloptions.
|
||||
$cache_cleared = wp_cache_flush();
|
||||
$rows = array();
|
||||
foreach ($current['columns'] as $item) {
|
||||
$rows[] = array('table' => $item['table'], 'column' => $item['column'], 'affected' => $item['count']);
|
||||
}
|
||||
return array('rows' => $rows, 'total' => $current['total'], 'cache_cleared' => $cache_cleared);
|
||||
}
|
||||
|
||||
private static function conversion($column)
|
||||
private static function repair_text($value)
|
||||
{
|
||||
$column_sql = self::identifier($column);
|
||||
return "CONVERT(BINARY CONVERT({$column_sql} USING latin1) USING utf8mb4)";
|
||||
static $map = null;
|
||||
if ($map === null) {
|
||||
$map = array();
|
||||
// The two common misdecodings: ISO-8859-1 and Windows-1252.
|
||||
$windows = array(
|
||||
0x80 => '€', 0x82 => '‚', 0x83 => 'ƒ', 0x84 => '„', 0x85 => '…',
|
||||
0x86 => '†', 0x87 => '‡', 0x88 => 'ˆ', 0x89 => '‰', 0x8a => 'Š',
|
||||
0x8b => '‹', 0x8c => 'Œ', 0x8e => 'Ž', 0x91 => '‘', 0x92 => '’',
|
||||
0x93 => '“', 0x94 => '”', 0x95 => '•', 0x96 => '–', 0x97 => '—',
|
||||
0x98 => '˜', 0x99 => '™', 0x9a => 'š', 0x9b => '›', 0x9c => 'œ',
|
||||
0x9e => 'ž', 0x9f => 'Ÿ',
|
||||
);
|
||||
foreach (preg_split('//u', 'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ', -1, PREG_SPLIT_NO_EMPTY) as $letter) {
|
||||
$latin = '';
|
||||
$cp1252 = '';
|
||||
foreach (unpack('C*', $letter) as $byte) {
|
||||
$encoded = $byte < 128 ? chr($byte) : chr(0xc0 | ($byte >> 6)) . chr(0x80 | ($byte & 0x3f));
|
||||
$latin .= $encoded;
|
||||
$cp1252 .= isset($windows[$byte]) ? $windows[$byte] : $encoded;
|
||||
}
|
||||
$map[$latin] = $letter;
|
||||
$map[$cp1252] = $letter;
|
||||
}
|
||||
}
|
||||
// Never transcode the whole value: valid Unicode, CSS and JSON stay intact.
|
||||
return strtr($value, $map);
|
||||
}
|
||||
|
||||
private static function difference($column)
|
||||
private static function repair_value($value, $depth = 0)
|
||||
{
|
||||
$column_sql = self::identifier($column);
|
||||
$converted = self::conversion($column);
|
||||
return "NOT (BINARY {$column_sql} <=> BINARY {$converted})";
|
||||
if (self::repair_text($value) === $value) {
|
||||
return $value;
|
||||
}
|
||||
if ($depth > 64 || preg_match('//u', $value) !== 1) {
|
||||
throw new RuntimeException('Unsupported depth or invalid UTF-8.');
|
||||
}
|
||||
if (!preg_match('/\A(?:[aObisdCREr]:|N;)/', ltrim($value))) {
|
||||
return self::repair_text($value);
|
||||
}
|
||||
$offset = 0;
|
||||
$result = self::repair_serialized($value, $offset, $depth + 1);
|
||||
if ($offset !== strlen($value)) {
|
||||
throw new RuntimeException('Malformed or unsupported serialized value.');
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
private static function repair_serialized($value, &$offset, $depth, $key = false)
|
||||
{
|
||||
if ($depth > 64) {
|
||||
throw new RuntimeException('Serialized value is too deeply nested.');
|
||||
}
|
||||
$start = $offset;
|
||||
$type = substr($value, $offset, 1);
|
||||
if ($key && $type !== 's' && $type !== 'i') {
|
||||
throw new RuntimeException('Unsupported serialized key.');
|
||||
}
|
||||
if ($type === 's' && preg_match('/\Gs:([0-9]+):"/', $value, $match, 0, $offset)) {
|
||||
$offset += strlen($match[0]);
|
||||
$length = (int) $match[1];
|
||||
if ($length > strlen($value) - $offset - 2 || substr($value, $offset + $length, 2) !== '";') {
|
||||
throw new RuntimeException('Serialized string length is invalid. Restore this value from backup.');
|
||||
}
|
||||
$text = substr($value, $offset, $length);
|
||||
$offset += $length + 2;
|
||||
// Keys are identifiers, and changing them could collide or break settings.
|
||||
$fixed = $key ? $text : self::repair_value($text, $depth + 1);
|
||||
return $fixed === $text ? substr($value, $start, $offset - $start) : 's:' . strlen($fixed) . ':"' . $fixed . '";';
|
||||
}
|
||||
if ($type === 'a' && preg_match('/\Ga:([0-9]+):\{/', $value, $match, 0, $offset)) {
|
||||
$offset += strlen($match[0]);
|
||||
$count = (int) $match[1];
|
||||
if ($count > strlen($value) - $offset) {
|
||||
throw new RuntimeException('Invalid serialized array length.');
|
||||
}
|
||||
$result = $match[0];
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
$result .= self::repair_serialized($value, $offset, $depth + 1, true);
|
||||
$result .= self::repair_serialized($value, $offset, $depth + 1);
|
||||
}
|
||||
if (substr($value, $offset, 1) !== '}') {
|
||||
throw new RuntimeException('Malformed serialized array.');
|
||||
}
|
||||
$offset++;
|
||||
return $result . '}';
|
||||
}
|
||||
if (preg_match('/\G(?:N;|b:[01];|i:-?[0-9]+;|d:(?:-?INF|NAN|[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[Ee][+-]?[0-9]+)?);)/', $value, $match, 0, $offset)) {
|
||||
$offset += strlen($match[0]);
|
||||
return $match[0];
|
||||
}
|
||||
// Never instantiate objects or rewrite references/custom serialization.
|
||||
throw new RuntimeException('Malformed or unsupported serialized value; left unchanged.');
|
||||
}
|
||||
|
||||
private static function identifier($name)
|
||||
@@ -292,7 +395,7 @@ final class WP_Mojibake_Repair_Temporary
|
||||
}
|
||||
}
|
||||
|
||||
private static function make_approval($counts)
|
||||
private static function make_approval($preview)
|
||||
{
|
||||
global $wpdb;
|
||||
$payload = wp_json_encode(array(
|
||||
@@ -300,7 +403,9 @@ final class WP_Mojibake_Repair_Temporary
|
||||
'database' => DB_NAME,
|
||||
'prefix' => $wpdb->prefix,
|
||||
'expires' => time() + self::APPROVAL_TTL,
|
||||
'counts' => $counts,
|
||||
'version' => self::REPAIR_VERSION,
|
||||
'counts' => $preview['counts'],
|
||||
'fingerprints' => $preview['fingerprints'],
|
||||
));
|
||||
$encoded = rtrim(strtr(base64_encode($payload), '+/', '-_'), '=');
|
||||
return $encoded . '.' . hash_hmac('sha256', $encoded, wp_salt('auth'));
|
||||
@@ -319,6 +424,9 @@ final class WP_Mojibake_Repair_Temporary
|
||||
$payload = json_decode(base64_decode(strtr($encoded, '-_', '+/'), true), true);
|
||||
|
||||
if (!is_array($payload)
|
||||
|| !isset($payload['version'], $payload['user'], $payload['database'], $payload['prefix'], $payload['expires'], $payload['fingerprints'])
|
||||
|| $payload['version'] !== self::REPAIR_VERSION
|
||||
|| !is_array($payload['fingerprints'])
|
||||
|| (int) $payload['user'] !== get_current_user_id()
|
||||
|| (string) $payload['database'] !== DB_NAME
|
||||
|| (string) $payload['prefix'] !== $wpdb->prefix
|
||||
@@ -334,7 +442,7 @@ final class WP_Mojibake_Repair_Temporary
|
||||
$expected_ids[] = $table . '.' . $column;
|
||||
}
|
||||
}
|
||||
if (array_keys($payload['counts']) !== $expected_ids) {
|
||||
if (array_keys($payload['counts']) !== $expected_ids || array_keys($payload['fingerprints']) !== $expected_ids) {
|
||||
throw new RuntimeException('The preview approval does not match the fixed repair plan.');
|
||||
}
|
||||
|
||||
@@ -356,16 +464,16 @@ final class WP_Mojibake_Repair_Temporary
|
||||
{
|
||||
echo '<h2>Preview</h2>';
|
||||
echo '<p><strong>' . number_format_i18n($preview['total']) . '</strong> differing column values found. Only the samples below will be displayed; the confirmed update covers all counted candidates.</p>';
|
||||
echo '<div class="notice notice-warning"><p><strong>Important:</strong> “different” does not prove mojibake. The conversion is not safe to run twice. Correct non-Latin-1 text can also change.</p></div>';
|
||||
if ($preview['serialized'] > 0) {
|
||||
echo '<div class="notice notice-error"><p><strong>' . number_format_i18n($preview['serialized']) . ' candidate values look PHP-serialized.</strong> SQL conversion can invalidate embedded string lengths. Review these samples carefully.</p></div>';
|
||||
echo '<p>Only recognized Czech mojibake sequences are replaced. Correct Unicode is preserved, and serialized string lengths are updated. Review the samples to confirm that these replacements match your content.</p>';
|
||||
if ($preview['skipped'] > 0) {
|
||||
echo '<div class="notice notice-warning"><p><strong>' . number_format_i18n($preview['skipped']) . ' values were skipped.</strong> They contain repair candidates but have malformed or unsupported data (such as serialized objects or references). Already broken settings need recovery from a backup.</p></div>';
|
||||
}
|
||||
|
||||
foreach ($preview['columns'] as $item) {
|
||||
echo '<details style="margin:12px 0;padding:10px;background:#fff;border:1px solid #ccd0d4"' . ($item['count'] ? ' open' : '') . '>';
|
||||
echo '<summary><code>' . esc_html($item['table'] . '.' . $item['column']) . '</code> — ' . number_format_i18n($item['count']) . ' differing</summary>';
|
||||
if ($item['serialized']) {
|
||||
echo '<p><strong>Serialized-looking: ' . number_format_i18n($item['serialized']) . '</strong></p>';
|
||||
if ($item['skipped']) {
|
||||
echo '<p><strong>Skipped: ' . number_format_i18n($item['skipped']) . '</strong>. Example row IDs: ' . esc_html(implode(', ', $item['skipped_ids'])) . '</p>';
|
||||
}
|
||||
foreach ($item['samples'] as $sample) {
|
||||
echo '<p><strong>Row ' . esc_html((string) $sample['row_id']) . '</strong></p>';
|
||||
@@ -375,22 +483,26 @@ final class WP_Mojibake_Repair_Temporary
|
||||
}
|
||||
|
||||
if ($preview['total'] < 1) {
|
||||
echo '<div class="notice notice-success"><p>No update is needed.</p></div>';
|
||||
echo '<div class="notice notice-success"><p>No supported repair candidates were found.</p></div>';
|
||||
return;
|
||||
}
|
||||
|
||||
echo '<h2>Step 2: Run once</h2><form method="post">';
|
||||
wp_nonce_field('wpmr_action', 'wpmr_nonce');
|
||||
echo '<input type="hidden" name="wpmr_action" value="execute">';
|
||||
echo '<input type="hidden" name="approval" value="' . esc_attr(self::make_approval($preview['counts'])) . '">';
|
||||
echo '<input type="hidden" name="approval" value="' . esc_attr(self::make_approval($preview)) . '">';
|
||||
echo '<label><input type="checkbox" name="confirm_backup" value="1" required> I have a restorable backup and approve updating every counted candidate above.</label>';
|
||||
submit_button('Run repair once', 'primary', 'submit', false, array('onclick' => "return confirm('Run the repair now? This must not be run twice.');"));
|
||||
submit_button('Run repair once', 'primary', 'submit', false, array('onclick' => "return confirm('Run the repair now? This updates all approved candidates.');"));
|
||||
echo '</form>';
|
||||
}
|
||||
|
||||
private static function render_result($result)
|
||||
{
|
||||
echo '<div class="notice notice-success"><p><strong>Repair committed successfully.</strong> ' . number_format_i18n($result['total']) . ' column values updated.</p></div>';
|
||||
if (!$result['cache_cleared']) {
|
||||
echo '<div class="notice notice-warning"><p>The database repair succeeded, but the WordPress object cache could not be cleared. Clear it through your hosting provider.</p></div>';
|
||||
}
|
||||
echo '<p>Clear any page cache and regenerate your page builder’s CSS before checking the site.</p>';
|
||||
echo '<table class="widefat striped" style="max-width:900px"><thead><tr><th>Table</th><th>Column</th><th>Affected</th></tr></thead><tbody>';
|
||||
foreach ($result['rows'] as $row) {
|
||||
echo '<tr><td><code>' . esc_html($row['table']) . '</code></td><td><code>' . esc_html($row['column']) . '</code></td><td>' . number_format_i18n($row['affected']) . '</td></tr>';
|
||||
|
||||
Reference in New Issue
Block a user