diff options
author | Brian Evans <grknight@gentoo.org> | 2018-11-20 10:51:06 -0500 |
---|---|---|
committer | Brian Evans <grknight@gentoo.org> | 2018-11-20 10:51:06 -0500 |
commit | 1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb (patch) | |
tree | ad113bd05db878a61b503938c05fe046eca25ee0 /MLEB/Translate/utils/MessageIndex.php | |
parent | LinkAttributes: Update to v0.2 (diff) | |
download | extensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.tar.gz extensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.tar.bz2 extensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.zip |
Update to MediaWikiLanguageExtensionBundle-2018.10
Signed-off-by: Brian Evans <grknight@gentoo.org>
Diffstat (limited to 'MLEB/Translate/utils/MessageIndex.php')
-rw-r--r-- | MLEB/Translate/utils/MessageIndex.php | 422 |
1 files changed, 329 insertions, 93 deletions
diff --git a/MLEB/Translate/utils/MessageIndex.php b/MLEB/Translate/utils/MessageIndex.php index 98ca8128..6d0bb520 100644 --- a/MLEB/Translate/utils/MessageIndex.php +++ b/MLEB/Translate/utils/MessageIndex.php @@ -5,7 +5,7 @@ * @file * @author Niklas Laxstrom * @copyright Copyright © 2008-2013, Niklas Laxström - * @license GPL-2.0+ + * @license GPL-2.0-or-later */ /** @@ -17,11 +17,18 @@ * to message groups. */ abstract class MessageIndex { - /// @var MessageIndex + /** + * @var self + */ protected static $instance; /** - * @return MessageIndex + * @var MapCacheLRU|null + */ + private static $keysCache; + + /** + * @return self */ public static function singleton() { if ( self::$instance === null ) { @@ -35,22 +42,55 @@ abstract class MessageIndex { } /** + * Override the global instance, for testing. + * + * @since 2015.04 + * @param MessageIndex $instance + */ + public static function setInstance( self $instance ) { + self::$instance = $instance; + } + + /** * Retrieves a list of groups given MessageHandle belongs to. * @since 2012-01-04 * @param MessageHandle $handle * @return array */ public static function getGroupIds( MessageHandle $handle ) { - $namespace = $handle->getTitle()->getNamespace(); + global $wgTranslateMessageNamespaces; + + $title = $handle->getTitle(); + + if ( !$title->inNamespaces( $wgTranslateMessageNamespaces ) ) { + return []; + } + + $namespace = $title->getNamespace(); $key = $handle->getKey(); $normkey = TranslateUtils::normaliseKey( $namespace, $key ); - $value = self::singleton()->get( $normkey ); - if ( $value !== null ) { - return (array)$value; - } else { - return array(); + $cache = self::getCache(); + $value = $cache->get( $normkey ); + if ( $value === null ) { + $value = self::singleton()->get( $normkey ); + $value = $value !== null + ? (array)$value + : []; + $cache->set( $normkey, $value ); } + + return $value; + } + + /** + * @return MapCacheLRU + */ + private static function getCache() { + if ( self::$keysCache === null ) { + self::$keysCache = new MapCacheLRU( 30 ); + } + return self::$keysCache; } /** @@ -80,10 +120,29 @@ abstract class MessageIndex { } } - /// @return array - abstract public function retrieve(); + /** + * @param bool $forRebuild + * @return array + */ + abstract public function retrieve( $forRebuild = false ); + + /** + * @since 2018.01 + * @return string[] + */ + public function getKeys() { + return array_keys( $this->retrieve() ); + } + + abstract protected function store( array $array, array $diff ); - abstract protected function store( array $array ); + protected function lock() { + return true; + } + + protected function unlock() { + return true; + } public function rebuild() { static $recursion = 0; @@ -93,21 +152,29 @@ abstract class MessageIndex { wfWarn( $msg ); $recursion--; - return array(); + return []; } $recursion++; $groups = MessageGroups::singleton()->getGroups(); - $new = $old = array(); - $old = $this->retrieve(); - $postponed = array(); + if ( !$this->lock() ) { + throw new Exception( __CLASS__ . ': unable to acquire lock' ); + } + + self::getCache()->clear(); + + $new = $old = []; + $old = $this->retrieve( 'rebuild' ); + $postponed = []; /** * @var MessageGroup $g */ foreach ( $groups as $g ) { if ( !$g->exists() ) { + $id = $g->getId(); + wfWarn( __METHOD__ . ": group '$id' is registered but does not exist" ); continue; } @@ -124,61 +191,106 @@ abstract class MessageIndex { $this->checkAndAdd( $new, $g, true ); } - $this->store( $new ); - $this->clearMessageGroupStats( $old, $new ); + $diff = self::getArrayDiff( $old, $new ); + $this->store( $new, $diff['keys'] ); + $this->unlock(); + $this->clearMessageGroupStats( $diff ); + $recursion--; return $new; } /** - * Purge message group stats when set of keys have changed. + * Compares two associative arrays. + * + * Values must be a string or list of strings. Returns an array of added, + * deleted and modified keys as well as value changes (you can think values + * as categories and keys as pages). Each of the keys ('add', 'del', 'mod' + * respectively) maps to an array whose keys are the changed keys of the + * original arrays and values are lists where first element contains the + * old value and the second element the new value. + * + * @code + * $a = [ 'a' => '1', 'b' => '2', 'c' => '3' ]; + * $b = [ 'b' => '2', 'c' => [ '3', '2' ], 'd' => '4' ]; + * + * self::getArrayDiff( $a, $b ) ) === [ + * 'keys' => [ + * 'add' => [ 'd' => [ [], [ '4' ] ] ], + * 'del' => [ 'a' => [ [ '1' ], [] ] ], + * 'mod' => [ 'c' => [ [ '3' ], [ '3', '2' ] ] ], + * ], + * 'values' => [ 2, 4, 1 ] + * ]; + * @endcode + * * @param array $old * @param array $new + * @return array */ - protected function clearMessageGroupStats( array $old, array $new ) { - $changes = array(); + public static function getArrayDiff( array $old, array $new ) { + $values = []; + $record = function ( $groups ) use ( &$values ) { + foreach ( $groups as $group ) { + $values[$group] = true; + } + }; + + $keys = [ + 'add' => [], + 'del' => [], + 'mod' => [], + ]; foreach ( $new as $key => $groups ) { - // Using != here on purpose to ignore order of items if ( !isset( $old[$key] ) ) { - $changes[$key] = array( array(), (array)$groups ); + $keys['add'][$key] = [ [], (array)$groups ]; + $record( (array)$groups ); + // Using != here on purpose to ignore the order of items } elseif ( $groups != $old[$key] ) { - $changes[$key] = array( (array)$old[$key], (array)$groups ); + $keys['mod'][$key] = [ (array)$old[$key], (array)$groups ]; + $record( array_diff( (array)$old[$key], (array)$groups ) ); + $record( array_diff( (array)$groups, (array)$old[$key] ) ); } } foreach ( $old as $key => $groups ) { if ( !isset( $new[$key] ) ) { - $changes[$key] = array( (array)$groups, array() ); + $keys['del'][$key] = [ (array)$groups, [] ]; + $record( (array)$groups, [] ); } // We already checked for diffs above } - $changedGroups = array(); - foreach ( $changes as $data ) { - foreach ( $data[0] as $group ) { - $changedGroups[$group] = true; - } - foreach ( $data[1] as $group ) { - $changedGroups[$group] = true; - } - } - - MessageGroupStats::clearGroup( array_keys( $changedGroups ) ); + return [ + 'keys' => $keys, + 'values' => array_keys( $values ), + ]; + } - foreach ( $changes as $key => $data ) { - list( $ns, $pagename ) = explode( ':', $key, 2 ); - $title = Title::makeTitle( $ns, $pagename ); - $handle = new MessageHandle( $title ); - list ( $oldGroups, $newGroups ) = $data; - wfRunHooks( 'TranslateEventMessageMembershipChange', - array( $handle, $oldGroups, $newGroups ) ); + /** + * Purge stuff when set of keys have changed. + * + * @param array $diff + */ + protected function clearMessageGroupStats( array $diff ) { + MessageGroupStats::forGroup( $diff['values'], MessageGroupStats::FLAG_NO_CACHE ); + + foreach ( $diff['keys'] as $keys ) { + foreach ( $keys as $key => $data ) { + list( $ns, $pagename ) = explode( ':', $key, 2 ); + $title = Title::makeTitle( $ns, $pagename ); + $handle = new MessageHandle( $title ); + list( $oldGroups, $newGroups ) = $data; + Hooks::run( 'TranslateEventMessageMembershipChange', + [ $handle, $oldGroups, $newGroups ] ); + } } } /** - * @param array $hugearray + * @param array &$hugearray * @param MessageGroup $g * @param bool $ignore */ @@ -219,7 +331,7 @@ abstract class MessageIndex { // references instead. References are hard! $value = & $hugearray[$key]; unset( $hugearray[$key] ); - $hugearray[$key] = array( &$value, &$id ); + $hugearray[$key] = [ &$value, &$id ]; } } else { $hugearray[$key] = & $id; @@ -228,9 +340,13 @@ abstract class MessageIndex { unset( $id ); // Disconnect the previous references to this $id } - /* These are probably slower than serialize and unserialize, + /** + * These are probably slower than serialize and unserialize, * but they are more space efficient because we only need - * strings and arrays. */ + * strings and arrays. + * @param mixed $data + * @return mixed + */ protected function serialize( $data ) { if ( is_array( $data ) ) { return implode( '|', $data ); @@ -263,13 +379,18 @@ abstract class MessageIndex { * which provides random access - this backend doesn't support that. */ class SerializedMessageIndex extends MessageIndex { - /// @var array + /** + * @var array|null + */ protected $index; protected $filename = 'translate_messageindex.ser'; - /** @return array */ - public function retrieve() { + /** + * @param bool $forRebuild + * @return array + */ + public function retrieve( $forRebuild = false ) { if ( $this->index !== null ) { return $this->index; } @@ -284,7 +405,7 @@ class SerializedMessageIndex extends MessageIndex { return $this->index; } - protected function store( array $array ) { + protected function store( array $array, array $diff ) { $file = TranslateUtils::cacheFile( $this->filename ); file_put_contents( $file, serialize( $array ) ); $this->index = $array; @@ -307,18 +428,55 @@ class FileCachedMessageIndex extends SerializedMessageIndex { * @since 2012-04-12 */ class DatabaseMessageIndex extends MessageIndex { - /// @var array + /** + * @var array|null + */ protected $index; - /** @return array */ - public function retrieve() { - if ( $this->index !== null ) { + protected function lock() { + $dbw = wfGetDB( DB_MASTER ); + + // Any transaction should be flushed after getting the lock to avoid + // stale pre-lock REPEATABLE-READ snapshot data. + $ok = $dbw->lock( 'translate-messageindex', __METHOD__, 30 ); + if ( $ok ) { + $dbw->commit( __METHOD__, 'flush' ); + } + + return $ok; + } + + protected function unlock() { + $fname = __METHOD__; + $dbw = wfGetDB( DB_MASTER ); + // Unlock once the rows are actually unlocked to avoid deadlocks + if ( !$dbw->trxLevel() ) { + $dbw->unlock( 'translate-messageindex', $fname ); + } elseif ( method_exists( $dbw, 'onTransactionResolution' ) ) { // 1.28 + $dbw->onTransactionResolution( function () use ( $dbw, $fname ) { + $dbw->unlock( 'translate-messageindex', $fname ); + } ); + } else { + $dbw->onTransactionIdle( function () use ( $dbw, $fname ) { + $dbw->unlock( 'translate-messageindex', $fname ); + } ); + } + + return true; + } + + /** + * @param bool $forRebuild + * @return array + */ + public function retrieve( $forRebuild = false ) { + if ( $this->index !== null && !$forRebuild ) { return $this->index; } - $dbr = wfGetDB( DB_SLAVE ); - $res = $dbr->select( 'translate_messageindex', '*', array(), __METHOD__ ); - $this->index = array(); + $dbr = wfGetDB( $forRebuild ? DB_MASTER : DB_REPLICA ); + $res = $dbr->select( 'translate_messageindex', '*', [], __METHOD__ ); + $this->index = []; foreach ( $res as $row ) { $this->index[$row->tmi_key] = $this->unserialize( $row->tmi_value ); } @@ -327,11 +485,11 @@ class DatabaseMessageIndex extends MessageIndex { } protected function get( $key ) { - $dbr = wfGetDB( DB_SLAVE ); + $dbr = wfGetDB( DB_REPLICA ); $value = $dbr->selectField( 'translate_messageindex', 'tmi_value', - array( 'tmi_key' => $key ), + [ 'tmi_key' => $key ], __METHOD__ ); @@ -344,25 +502,35 @@ class DatabaseMessageIndex extends MessageIndex { return $value; } - protected function store( array $array ) { - $dbw = wfGetDB( DB_MASTER ); - $rows = array(); + protected function store( array $array, array $diff ) { + $updates = []; - foreach ( $array as $key => $value ) { - $value = $this->serialize( $value ); - $rows[] = array( 'tmi_key' => $key, 'tmi_value' => $value ); + foreach ( [ $diff['add'], $diff['mod'] ] as $changes ) { + foreach ( $changes as $key => $data ) { + list( $old, $new ) = $data; + $updates[] = [ + 'tmi_key' => $key, + 'tmi_value' => $this->serialize( $new ), + ]; + } } - // BC for <= MW 1.22 - if ( method_exists( $dbw, 'startAtomic' ) ) { - $dbw->startAtomic( __METHOD__ ); + $index = [ 'tmi_key' ]; + $deletions = array_keys( $diff['del'] ); + + $dbw = wfGetDB( DB_MASTER ); + $dbw->startAtomic( __METHOD__ ); + + if ( $updates !== [] ) { + $dbw->replace( 'translate_messageindex', [ $index ], $updates, __METHOD__ ); } - $dbw->delete( 'translate_messageindex', '*', __METHOD__ ); - $dbw->insert( 'translate_messageindex', $rows, __METHOD__ ); - if ( method_exists( $dbw, 'endAtomic' ) ) { - $dbw->endAtomic( __METHOD__ ); + + if ( $deletions !== [] ) { + $dbw->delete( 'translate_messageindex', [ 'tmi_key' => $deletions ], __METHOD__ ); } + $dbw->endAtomic( __METHOD__ ); + $this->index = $array; } } @@ -379,20 +547,25 @@ class CachedMessageIndex extends MessageIndex { protected $key = 'translate-messageindex'; protected $cache; - /// @var array + /** + * @var array|null + */ protected $index; protected function __construct( array $params ) { $this->cache = wfGetCache( CACHE_ANYTHING ); } - /** @return array */ - public function retrieve() { + /** + * @param bool $forRebuild + * @return array + */ + public function retrieve( $forRebuild = false ) { if ( $this->index !== null ) { return $this->index; } - $key = wfMemckey( $this->key ); + $key = wfMemcKey( $this->key ); $data = $this->cache->get( $key ); if ( is_array( $data ) ) { $this->index = $data; @@ -403,8 +576,8 @@ class CachedMessageIndex extends MessageIndex { return $this->index; } - protected function store( array $array ) { - $key = wfMemckey( $this->key ); + protected function store( array $array, array $diff ) { + $key = wfMemcKey( $this->key ); $this->cache->set( $key, $array ); $this->index = $array; @@ -425,32 +598,54 @@ class CachedMessageIndex extends MessageIndex { * @since 2012-04-10 */ class CDBMessageIndex extends MessageIndex { - /// @var array + /** + * @var array|null + */ protected $index; - /// @var CdbReader + /** + * @var \Cdb\Reader|null + */ protected $reader; - /// @var string + /** + * @var string + */ protected $filename = 'translate_messageindex.cdb'; - /** @return array */ - public function retrieve() { + /** + * @param bool $forRebuild + * @return array + */ + public function retrieve( $forRebuild = false ) { $reader = $this->getReader(); // This must be below the line above, which may fill the index if ( $this->index !== null ) { return $this->index; } - $keys = (array)$this->unserialize( $reader->get( '#keys' ) ); - $this->index = array(); - foreach ( $keys as $key ) { + $this->index = []; + foreach ( $this->getKeys() as $key ) { $this->index[$key] = $this->unserialize( $reader->get( $key ) ); } return $this->index; } + public function getKeys() { + $reader = $this->getReader(); + $keys = []; + while ( true ) { + $key = $keys === [] ? $reader->firstkey() : $reader->nextkey(); + if ( $key === false ) { + break; + } + $keys[] = $key; + } + + return $keys; + } + protected function get( $key ) { $reader = $this->getReader(); // We might have the full cache loaded @@ -472,13 +667,11 @@ class CDBMessageIndex extends MessageIndex { return $value; } - protected function store( array $array ) { + protected function store( array $array, array $diff ) { $this->reader = null; $file = TranslateUtils::cacheFile( $this->filename ); - $cache = CdbWriter::open( $file ); - $keys = array_keys( $array ); - $cache->set( '#keys', $this->serialize( $keys ) ); + $cache = \Cdb\Writer::open( $file ); foreach ( $array as $key => $value ) { $value = $this->serialize( $value ); @@ -498,10 +691,53 @@ class CDBMessageIndex extends MessageIndex { $file = TranslateUtils::cacheFile( $this->filename ); if ( !file_exists( $file ) ) { // Create an empty index to allow rebuild - $this->store( array() ); + $this->store( [], [] ); $this->index = $this->rebuild(); } - return $this->reader = CdbReader::open( $file ); + $this->reader = \Cdb\Reader::open( $file ); + return $this->reader; + } +} + +/** + * Storage on hash. + * + * For testing. + * + * @since 2015.04 + */ +class HashMessageIndex extends MessageIndex { + /** + * @var array + */ + protected $index = []; + + /** + * @param bool $forRebuild + * @return array + */ + public function retrieve( $forRebuild = false ) { + return $this->index; + } + + /** + * @param string $key + * + * @return mixed + */ + protected function get( $key ) { + if ( isset( $this->index[$key] ) ) { + return $this->index[$key]; + } else { + return null; + } + } + + protected function store( array $array, array $diff ) { + $this->index = $array; + } + + protected function clearMessageGroupStats( array $diff ) { } } |