summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBrian Evans <grknight@gentoo.org>2018-11-20 10:51:06 -0500
committerBrian Evans <grknight@gentoo.org>2018-11-20 10:51:06 -0500
commit1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb (patch)
treead113bd05db878a61b503938c05fe046eca25ee0 /MLEB/Translate/utils/MessageIndex.php
parentLinkAttributes: Update to v0.2 (diff)
downloadextensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.tar.gz
extensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.tar.bz2
extensions-1ea74fa59d8d1c6c12d20be6c8e7d5ac7f370fdb.zip
Update to MediaWikiLanguageExtensionBundle-2018.10
Signed-off-by: Brian Evans <grknight@gentoo.org>
Diffstat (limited to 'MLEB/Translate/utils/MessageIndex.php')
-rw-r--r--MLEB/Translate/utils/MessageIndex.php422
1 files changed, 329 insertions, 93 deletions
diff --git a/MLEB/Translate/utils/MessageIndex.php b/MLEB/Translate/utils/MessageIndex.php
index 98ca8128..6d0bb520 100644
--- a/MLEB/Translate/utils/MessageIndex.php
+++ b/MLEB/Translate/utils/MessageIndex.php
@@ -5,7 +5,7 @@
* @file
* @author Niklas Laxstrom
* @copyright Copyright © 2008-2013, Niklas Laxström
- * @license GPL-2.0+
+ * @license GPL-2.0-or-later
*/
/**
@@ -17,11 +17,18 @@
* to message groups.
*/
abstract class MessageIndex {
- /// @var MessageIndex
+ /**
+ * @var self
+ */
protected static $instance;
/**
- * @return MessageIndex
+ * @var MapCacheLRU|null
+ */
+ private static $keysCache;
+
+ /**
+ * @return self
*/
public static function singleton() {
if ( self::$instance === null ) {
@@ -35,22 +42,55 @@ abstract class MessageIndex {
}
/**
+ * Override the global instance, for testing.
+ *
+ * @since 2015.04
+ * @param MessageIndex $instance
+ */
+ public static function setInstance( self $instance ) {
+ self::$instance = $instance;
+ }
+
+ /**
* Retrieves a list of groups given MessageHandle belongs to.
* @since 2012-01-04
* @param MessageHandle $handle
* @return array
*/
public static function getGroupIds( MessageHandle $handle ) {
- $namespace = $handle->getTitle()->getNamespace();
+ global $wgTranslateMessageNamespaces;
+
+ $title = $handle->getTitle();
+
+ if ( !$title->inNamespaces( $wgTranslateMessageNamespaces ) ) {
+ return [];
+ }
+
+ $namespace = $title->getNamespace();
$key = $handle->getKey();
$normkey = TranslateUtils::normaliseKey( $namespace, $key );
- $value = self::singleton()->get( $normkey );
- if ( $value !== null ) {
- return (array)$value;
- } else {
- return array();
+ $cache = self::getCache();
+ $value = $cache->get( $normkey );
+ if ( $value === null ) {
+ $value = self::singleton()->get( $normkey );
+ $value = $value !== null
+ ? (array)$value
+ : [];
+ $cache->set( $normkey, $value );
}
+
+ return $value;
+ }
+
+ /**
+ * @return MapCacheLRU
+ */
+ private static function getCache() {
+ if ( self::$keysCache === null ) {
+ self::$keysCache = new MapCacheLRU( 30 );
+ }
+ return self::$keysCache;
}
/**
@@ -80,10 +120,29 @@ abstract class MessageIndex {
}
}
- /// @return array
- abstract public function retrieve();
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ abstract public function retrieve( $forRebuild = false );
+
+ /**
+ * @since 2018.01
+ * @return string[]
+ */
+ public function getKeys() {
+ return array_keys( $this->retrieve() );
+ }
+
+ abstract protected function store( array $array, array $diff );
- abstract protected function store( array $array );
+ protected function lock() {
+ return true;
+ }
+
+ protected function unlock() {
+ return true;
+ }
public function rebuild() {
static $recursion = 0;
@@ -93,21 +152,29 @@ abstract class MessageIndex {
wfWarn( $msg );
$recursion--;
- return array();
+ return [];
}
$recursion++;
$groups = MessageGroups::singleton()->getGroups();
- $new = $old = array();
- $old = $this->retrieve();
- $postponed = array();
+ if ( !$this->lock() ) {
+ throw new Exception( __CLASS__ . ': unable to acquire lock' );
+ }
+
+ self::getCache()->clear();
+
+ $new = $old = [];
+ $old = $this->retrieve( 'rebuild' );
+ $postponed = [];
/**
* @var MessageGroup $g
*/
foreach ( $groups as $g ) {
if ( !$g->exists() ) {
+ $id = $g->getId();
+ wfWarn( __METHOD__ . ": group '$id' is registered but does not exist" );
continue;
}
@@ -124,61 +191,106 @@ abstract class MessageIndex {
$this->checkAndAdd( $new, $g, true );
}
- $this->store( $new );
- $this->clearMessageGroupStats( $old, $new );
+ $diff = self::getArrayDiff( $old, $new );
+ $this->store( $new, $diff['keys'] );
+ $this->unlock();
+ $this->clearMessageGroupStats( $diff );
+
$recursion--;
return $new;
}
/**
- * Purge message group stats when set of keys have changed.
+ * Compares two associative arrays.
+ *
+ * Values must be a string or list of strings. Returns an array of added,
+ * deleted and modified keys as well as value changes (you can think values
+ * as categories and keys as pages). Each of the keys ('add', 'del', 'mod'
+ * respectively) maps to an array whose keys are the changed keys of the
+ * original arrays and values are lists where first element contains the
+ * old value and the second element the new value.
+ *
+ * @code
+ * $a = [ 'a' => '1', 'b' => '2', 'c' => '3' ];
+ * $b = [ 'b' => '2', 'c' => [ '3', '2' ], 'd' => '4' ];
+ *
+ * self::getArrayDiff( $a, $b ) ) === [
+ * 'keys' => [
+ * 'add' => [ 'd' => [ [], [ '4' ] ] ],
+ * 'del' => [ 'a' => [ [ '1' ], [] ] ],
+ * 'mod' => [ 'c' => [ [ '3' ], [ '3', '2' ] ] ],
+ * ],
+ * 'values' => [ 2, 4, 1 ]
+ * ];
+ * @endcode
+ *
* @param array $old
* @param array $new
+ * @return array
*/
- protected function clearMessageGroupStats( array $old, array $new ) {
- $changes = array();
+ public static function getArrayDiff( array $old, array $new ) {
+ $values = [];
+ $record = function ( $groups ) use ( &$values ) {
+ foreach ( $groups as $group ) {
+ $values[$group] = true;
+ }
+ };
+
+ $keys = [
+ 'add' => [],
+ 'del' => [],
+ 'mod' => [],
+ ];
foreach ( $new as $key => $groups ) {
- // Using != here on purpose to ignore order of items
if ( !isset( $old[$key] ) ) {
- $changes[$key] = array( array(), (array)$groups );
+ $keys['add'][$key] = [ [], (array)$groups ];
+ $record( (array)$groups );
+ // Using != here on purpose to ignore the order of items
} elseif ( $groups != $old[$key] ) {
- $changes[$key] = array( (array)$old[$key], (array)$groups );
+ $keys['mod'][$key] = [ (array)$old[$key], (array)$groups ];
+ $record( array_diff( (array)$old[$key], (array)$groups ) );
+ $record( array_diff( (array)$groups, (array)$old[$key] ) );
}
}
foreach ( $old as $key => $groups ) {
if ( !isset( $new[$key] ) ) {
- $changes[$key] = array( (array)$groups, array() );
+ $keys['del'][$key] = [ (array)$groups, [] ];
+ $record( (array)$groups, [] );
}
// We already checked for diffs above
}
- $changedGroups = array();
- foreach ( $changes as $data ) {
- foreach ( $data[0] as $group ) {
- $changedGroups[$group] = true;
- }
- foreach ( $data[1] as $group ) {
- $changedGroups[$group] = true;
- }
- }
-
- MessageGroupStats::clearGroup( array_keys( $changedGroups ) );
+ return [
+ 'keys' => $keys,
+ 'values' => array_keys( $values ),
+ ];
+ }
- foreach ( $changes as $key => $data ) {
- list( $ns, $pagename ) = explode( ':', $key, 2 );
- $title = Title::makeTitle( $ns, $pagename );
- $handle = new MessageHandle( $title );
- list ( $oldGroups, $newGroups ) = $data;
- wfRunHooks( 'TranslateEventMessageMembershipChange',
- array( $handle, $oldGroups, $newGroups ) );
+ /**
+ * Purge stuff when set of keys have changed.
+ *
+ * @param array $diff
+ */
+ protected function clearMessageGroupStats( array $diff ) {
+ MessageGroupStats::forGroup( $diff['values'], MessageGroupStats::FLAG_NO_CACHE );
+
+ foreach ( $diff['keys'] as $keys ) {
+ foreach ( $keys as $key => $data ) {
+ list( $ns, $pagename ) = explode( ':', $key, 2 );
+ $title = Title::makeTitle( $ns, $pagename );
+ $handle = new MessageHandle( $title );
+ list( $oldGroups, $newGroups ) = $data;
+ Hooks::run( 'TranslateEventMessageMembershipChange',
+ [ $handle, $oldGroups, $newGroups ] );
+ }
}
}
/**
- * @param array $hugearray
+ * @param array &$hugearray
* @param MessageGroup $g
* @param bool $ignore
*/
@@ -219,7 +331,7 @@ abstract class MessageIndex {
// references instead. References are hard!
$value = & $hugearray[$key];
unset( $hugearray[$key] );
- $hugearray[$key] = array( &$value, &$id );
+ $hugearray[$key] = [ &$value, &$id ];
}
} else {
$hugearray[$key] = & $id;
@@ -228,9 +340,13 @@ abstract class MessageIndex {
unset( $id ); // Disconnect the previous references to this $id
}
- /* These are probably slower than serialize and unserialize,
+ /**
+ * These are probably slower than serialize and unserialize,
* but they are more space efficient because we only need
- * strings and arrays. */
+ * strings and arrays.
+ * @param mixed $data
+ * @return mixed
+ */
protected function serialize( $data ) {
if ( is_array( $data ) ) {
return implode( '|', $data );
@@ -263,13 +379,18 @@ abstract class MessageIndex {
* which provides random access - this backend doesn't support that.
*/
class SerializedMessageIndex extends MessageIndex {
- /// @var array
+ /**
+ * @var array|null
+ */
protected $index;
protected $filename = 'translate_messageindex.ser';
- /** @return array */
- public function retrieve() {
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ public function retrieve( $forRebuild = false ) {
if ( $this->index !== null ) {
return $this->index;
}
@@ -284,7 +405,7 @@ class SerializedMessageIndex extends MessageIndex {
return $this->index;
}
- protected function store( array $array ) {
+ protected function store( array $array, array $diff ) {
$file = TranslateUtils::cacheFile( $this->filename );
file_put_contents( $file, serialize( $array ) );
$this->index = $array;
@@ -307,18 +428,55 @@ class FileCachedMessageIndex extends SerializedMessageIndex {
* @since 2012-04-12
*/
class DatabaseMessageIndex extends MessageIndex {
- /// @var array
+ /**
+ * @var array|null
+ */
protected $index;
- /** @return array */
- public function retrieve() {
- if ( $this->index !== null ) {
+ protected function lock() {
+ $dbw = wfGetDB( DB_MASTER );
+
+ // Any transaction should be flushed after getting the lock to avoid
+ // stale pre-lock REPEATABLE-READ snapshot data.
+ $ok = $dbw->lock( 'translate-messageindex', __METHOD__, 30 );
+ if ( $ok ) {
+ $dbw->commit( __METHOD__, 'flush' );
+ }
+
+ return $ok;
+ }
+
+ protected function unlock() {
+ $fname = __METHOD__;
+ $dbw = wfGetDB( DB_MASTER );
+ // Unlock once the rows are actually unlocked to avoid deadlocks
+ if ( !$dbw->trxLevel() ) {
+ $dbw->unlock( 'translate-messageindex', $fname );
+ } elseif ( method_exists( $dbw, 'onTransactionResolution' ) ) { // 1.28
+ $dbw->onTransactionResolution( function () use ( $dbw, $fname ) {
+ $dbw->unlock( 'translate-messageindex', $fname );
+ } );
+ } else {
+ $dbw->onTransactionIdle( function () use ( $dbw, $fname ) {
+ $dbw->unlock( 'translate-messageindex', $fname );
+ } );
+ }
+
+ return true;
+ }
+
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ public function retrieve( $forRebuild = false ) {
+ if ( $this->index !== null && !$forRebuild ) {
return $this->index;
}
- $dbr = wfGetDB( DB_SLAVE );
- $res = $dbr->select( 'translate_messageindex', '*', array(), __METHOD__ );
- $this->index = array();
+ $dbr = wfGetDB( $forRebuild ? DB_MASTER : DB_REPLICA );
+ $res = $dbr->select( 'translate_messageindex', '*', [], __METHOD__ );
+ $this->index = [];
foreach ( $res as $row ) {
$this->index[$row->tmi_key] = $this->unserialize( $row->tmi_value );
}
@@ -327,11 +485,11 @@ class DatabaseMessageIndex extends MessageIndex {
}
protected function get( $key ) {
- $dbr = wfGetDB( DB_SLAVE );
+ $dbr = wfGetDB( DB_REPLICA );
$value = $dbr->selectField(
'translate_messageindex',
'tmi_value',
- array( 'tmi_key' => $key ),
+ [ 'tmi_key' => $key ],
__METHOD__
);
@@ -344,25 +502,35 @@ class DatabaseMessageIndex extends MessageIndex {
return $value;
}
- protected function store( array $array ) {
- $dbw = wfGetDB( DB_MASTER );
- $rows = array();
+ protected function store( array $array, array $diff ) {
+ $updates = [];
- foreach ( $array as $key => $value ) {
- $value = $this->serialize( $value );
- $rows[] = array( 'tmi_key' => $key, 'tmi_value' => $value );
+ foreach ( [ $diff['add'], $diff['mod'] ] as $changes ) {
+ foreach ( $changes as $key => $data ) {
+ list( $old, $new ) = $data;
+ $updates[] = [
+ 'tmi_key' => $key,
+ 'tmi_value' => $this->serialize( $new ),
+ ];
+ }
}
- // BC for <= MW 1.22
- if ( method_exists( $dbw, 'startAtomic' ) ) {
- $dbw->startAtomic( __METHOD__ );
+ $index = [ 'tmi_key' ];
+ $deletions = array_keys( $diff['del'] );
+
+ $dbw = wfGetDB( DB_MASTER );
+ $dbw->startAtomic( __METHOD__ );
+
+ if ( $updates !== [] ) {
+ $dbw->replace( 'translate_messageindex', [ $index ], $updates, __METHOD__ );
}
- $dbw->delete( 'translate_messageindex', '*', __METHOD__ );
- $dbw->insert( 'translate_messageindex', $rows, __METHOD__ );
- if ( method_exists( $dbw, 'endAtomic' ) ) {
- $dbw->endAtomic( __METHOD__ );
+
+ if ( $deletions !== [] ) {
+ $dbw->delete( 'translate_messageindex', [ 'tmi_key' => $deletions ], __METHOD__ );
}
+ $dbw->endAtomic( __METHOD__ );
+
$this->index = $array;
}
}
@@ -379,20 +547,25 @@ class CachedMessageIndex extends MessageIndex {
protected $key = 'translate-messageindex';
protected $cache;
- /// @var array
+ /**
+ * @var array|null
+ */
protected $index;
protected function __construct( array $params ) {
$this->cache = wfGetCache( CACHE_ANYTHING );
}
- /** @return array */
- public function retrieve() {
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ public function retrieve( $forRebuild = false ) {
if ( $this->index !== null ) {
return $this->index;
}
- $key = wfMemckey( $this->key );
+ $key = wfMemcKey( $this->key );
$data = $this->cache->get( $key );
if ( is_array( $data ) ) {
$this->index = $data;
@@ -403,8 +576,8 @@ class CachedMessageIndex extends MessageIndex {
return $this->index;
}
- protected function store( array $array ) {
- $key = wfMemckey( $this->key );
+ protected function store( array $array, array $diff ) {
+ $key = wfMemcKey( $this->key );
$this->cache->set( $key, $array );
$this->index = $array;
@@ -425,32 +598,54 @@ class CachedMessageIndex extends MessageIndex {
* @since 2012-04-10
*/
class CDBMessageIndex extends MessageIndex {
- /// @var array
+ /**
+ * @var array|null
+ */
protected $index;
- /// @var CdbReader
+ /**
+ * @var \Cdb\Reader|null
+ */
protected $reader;
- /// @var string
+ /**
+ * @var string
+ */
protected $filename = 'translate_messageindex.cdb';
- /** @return array */
- public function retrieve() {
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ public function retrieve( $forRebuild = false ) {
$reader = $this->getReader();
// This must be below the line above, which may fill the index
if ( $this->index !== null ) {
return $this->index;
}
- $keys = (array)$this->unserialize( $reader->get( '#keys' ) );
- $this->index = array();
- foreach ( $keys as $key ) {
+ $this->index = [];
+ foreach ( $this->getKeys() as $key ) {
$this->index[$key] = $this->unserialize( $reader->get( $key ) );
}
return $this->index;
}
+ public function getKeys() {
+ $reader = $this->getReader();
+ $keys = [];
+ while ( true ) {
+ $key = $keys === [] ? $reader->firstkey() : $reader->nextkey();
+ if ( $key === false ) {
+ break;
+ }
+ $keys[] = $key;
+ }
+
+ return $keys;
+ }
+
protected function get( $key ) {
$reader = $this->getReader();
// We might have the full cache loaded
@@ -472,13 +667,11 @@ class CDBMessageIndex extends MessageIndex {
return $value;
}
- protected function store( array $array ) {
+ protected function store( array $array, array $diff ) {
$this->reader = null;
$file = TranslateUtils::cacheFile( $this->filename );
- $cache = CdbWriter::open( $file );
- $keys = array_keys( $array );
- $cache->set( '#keys', $this->serialize( $keys ) );
+ $cache = \Cdb\Writer::open( $file );
foreach ( $array as $key => $value ) {
$value = $this->serialize( $value );
@@ -498,10 +691,53 @@ class CDBMessageIndex extends MessageIndex {
$file = TranslateUtils::cacheFile( $this->filename );
if ( !file_exists( $file ) ) {
// Create an empty index to allow rebuild
- $this->store( array() );
+ $this->store( [], [] );
$this->index = $this->rebuild();
}
- return $this->reader = CdbReader::open( $file );
+ $this->reader = \Cdb\Reader::open( $file );
+ return $this->reader;
+ }
+}
+
+/**
+ * Storage on hash.
+ *
+ * For testing.
+ *
+ * @since 2015.04
+ */
+class HashMessageIndex extends MessageIndex {
+ /**
+ * @var array
+ */
+ protected $index = [];
+
+ /**
+ * @param bool $forRebuild
+ * @return array
+ */
+ public function retrieve( $forRebuild = false ) {
+ return $this->index;
+ }
+
+ /**
+ * @param string $key
+ *
+ * @return mixed
+ */
+ protected function get( $key ) {
+ if ( isset( $this->index[$key] ) ) {
+ return $this->index[$key];
+ } else {
+ return null;
+ }
+ }
+
+ protected function store( array $array, array $diff ) {
+ $this->index = $array;
+ }
+
+ protected function clearMessageGroupStats( array $diff ) {
}
}