Index: kernel/classes/clusterfilehandlers/dbbackends/mysql.php =================================================================== --- kernel/classes/clusterfilehandlers/dbbackends/mysql.php (revision 25359) +++ kernel/classes/clusterfilehandlers/dbbackends/mysql.php (working copy) @@ -1664,6 +1664,35 @@ return ( $row[0] + $this->dbparams['cache_generation_timeout'] ) - time(); } + /** + * Returns the list of expired binary files (images + binaries) + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + $query = "SELECT name FROM " . TABLE_METADATA . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[0]}, {$limit[1]}"; + } + $res = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = mysql_fetch_row( $res ) ) + $filePathList[] = $row[0]; + + return $filePathList; + } + public $db = null; public $numQueries = 0; public $transactionCount = 0; Index: kernel/classes/clusterfilehandlers/ezdbfilehandler.php =================================================================== --- kernel/classes/clusterfilehandlers/ezdbfilehandler.php (revision 25359) +++ kernel/classes/clusterfilehandlers/ezdbfilehandler.php (working copy) @@ -1307,6 +1307,32 @@ } /** + * eZDFS does require binary purge. + * It does store files in DB and therefore doesn't remove files in real time + * + * @since 4.3.0 + */ + public function requiresBinaryPurge() + { + return true; + } + + /** + * Fetches the first $limit expired binary items from the DB + * + * @param array $limit A 2 items array( offset, limit ) + * + * @return array(eZClusterFileHandlerInterace) + * @since 4.3.0 + * + * @todo handle output using $cli or something + */ + public function fetchExpiredBinaryItems( $limit = array( 0, 100 ) ) + { + return $this->backend->expiredFilesList( array( 'image', 'binaryfile' ), $limit ); + } + + /** * Database backend class * @var eZDBFileHandlerMysqlBackend **/ Index: kernel/classes/clusterfilehandlers/ezfsfilehandler.php =================================================================== --- kernel/classes/clusterfilehandlers/ezfsfilehandler.php (revision 25359) +++ kernel/classes/clusterfilehandlers/ezfsfilehandler.php (working copy) @@ -1049,6 +1049,17 @@ return false; } + /** + * eZFS does not require binary purge. + * Files are stored on plain FS and removed using FS functions + * + * @since 4.3 + */ + public function requiresBinaryPurge() + { + return false; + } + public $metaData = null; public $filePath; } Index: kernel/private/interfaces/ezclusterfilehandlerinterface.php =================================================================== --- kernel/private/interfaces/ezclusterfilehandlerinterface.php (revision 25359) +++ kernel/private/interfaces/ezclusterfilehandlerinterface.php (working copy) @@ -392,5 +392,13 @@ * @return bool **/ public function requiresClusterizing(); + + /** + * This method indicates if the cluster file handler requires binary files + * to be purged in order to be physically deleted + * + * @since 4.3 + */ + public function requiresBinaryPurge(); } ?> \ No newline at end of file Index: kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php =================================================================== --- kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php (revision 25359) +++ kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php (working copy) @@ -1399,6 +1399,31 @@ } /** + * eZDFS does require binary purge. + * It does store files in DB + on NFS, and therefore doesn't remove files + * in real time + * + * @since 4.3 + */ + public function requiresBinaryPurge() + { + return true; + } + + /** + * Fetches the first $limit expired binary items from the DB + * + * @param array $limit A 2 items array( offset, limit ) + * + * @return array(filepath) + * @since 4.3.0 + */ + public function fetchExpiredBinaryItems( $limit = array( 0 , 100 ) ) + { + return self::$dbbackend->expiredFilesList( array( 'image', 'binaryfile' ), $limit ); + } + + /** * Database backend class * Provides metadata operations * @var eZDFSFileHandlerMySQLBackend Index: kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php =================================================================== --- kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php (revision 25359) +++ kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php (working copy) @@ -1647,6 +1647,35 @@ } /** + * Returns the list of expired binary files (images + binaries) + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + $query = "SELECT name FROM " . self::TABLE_METADATA . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[0]}, {$limit[1]}"; + } + $res = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = mysql_fetch_row( $res ) ) + $filePathList[] = $row[0]; + + return $filePathList; + } + + /** * DB connexion handle * @var handle **/ Index: kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php =================================================================== --- kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php (revision 25359) +++ kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php (working copy) @@ -773,6 +773,17 @@ } /** + * eZFS2 doesn't require purge as it already purges files in realtime + * (FS based) + * + * @since 4.3 + */ + public function requiresBinaryPurge() + { + return false; + } + + /** * holds the real file path. This is only used when we are generating a cache * file, in which case $filePath holds the generating cache file name, * and $realFilePath holds the real name Index: kernel/private/classes/ezscriptclusterpurge.php =================================================================== --- kernel/private/classes/ezscriptclusterpurge.php (revision 0) +++ kernel/private/classes/ezscriptclusterpurge.php (revision 0) @@ -0,0 +1,174 @@ +options = array( + 'dry-run' => false, + 'iteration-sleep' => 1, + 'iteration-limit' => 100, + 'memory-monitoring' => false, + ); + } + + /** + * Performs preliminary checks in order to ensure the process can be + * started: + * - does the active cluster handler require purging of binary files + * + * @return bool + */ + public static function isRequired() + { + $clusterHandler = eZClusterFileHandler::instance(); + $result = $clusterHandler->requiresBinaryPurge(); + + return $result; + } + + /** + * Executes the purge operation + * + * @todo Endless loop on fetch list. The expired items are returned over and over again + **/ + public function run() + { + $cli = eZCLI::instance(); + + if ( $this->optMemoryMonitoring == true ) + { + eZLog::rotateLog( self::LOG_FILE ); + $cli->notice( "Logging memory usage to " . self::LOG_FILE ); + } + + if ( $this->optIterationSleep > 0 ) + $sleep = ( $this->optIterationSleep * 1000000 ); + else + $sleep = false; + + $limit = array( 0, $this->optIterationLimit ); + + $cli->output( "Purging expired binary items:" ); + + self::monitor( "start" ); + + // Fetch a limited list of purge items from the handler itself + $clusterHandler = eZClusterFileHandler::instance(); + while( $filesList = $clusterHandler->fetchExpiredBinaryItems( $limit ) ) + { + self::monitor( "iteration start" ); + foreach( $filesList as $file ) + { + $cli->output( "- $file" ); + if ( $this->optDryRun == false ) + { + self::monitor( "purge" ); + $fh = eZClusterFileHandler::instance( $file ); + $fh->purge( false, false ); + unset( $fh ); + } + } + if ( $sleep !== false ) + usleep( $sleep ); + + // the offset only has to be increased in dry run mode + // since each batch is not deleted + if ( $this->optDryRun == true ) + { + $limit[0] += $limit[1]; + } + self::monitor( "iteration end" ); + } + + self::monitor( "end" ); + } + + public function __get( $propertyName ) + { + switch( $propertyName ) + { + case 'optDryRun': + { + return $this->options['dry-run']; + } break; + + // no sleep in dry-run, it's not nap time ! + case 'optIterationSleep': + { + if ( $this->optDryRun == true ) + return 0; + else + return $this->options['iteration-sleep']; + } break; + + case 'optIterationLimit': + { + return $this->options['iteration-limit']; + } break; + + case 'optMemoryMonitoring': + { + return $this->options['memory-monitoring']; + } break; + } + } + + /** + * @todo Add type & value check + */ + public function __set( $propertyName, $propertyValue ) + { + switch( $propertyName ) + { + case 'optDryRun': + { + $this->options['dry-run'] = $propertyValue; + } break; + + case 'optIterationSleep': + { + return $this->options['iteration-sleep'] = $propertyValue; + } break; + + case 'optIterationLimit': + { + $this->options['iteration-limit'] = $propertyValue; + } break; + + case 'optMemoryMonitoring': + { + $this->options['memory-monitoring'] = $propertyValue; + } break; + } + } + + public function monitor( $text ) + { + if ( $this->opt == true ) + { + eZLog::write( "mem [$text]: " . memory_get_usage(), self::LOG_FILE ); + } + } + + private $options = array(); + + const LOG_FILE = 'clusterbinarypurge.log'; +} +?> Index: settings/cronjob.ini =================================================================== --- settings/cronjob.ini (revision 25359) +++ settings/cronjob.ini (working copy) @@ -43,6 +43,9 @@ [CronjobPart-unlock] Scripts[]=unlock.php +[CronjobPart-cluster_maintenance] +Scripts[]=clusterbinarypurge.php + # Example of a cronjob part # This one will only run the workflow cronjob script # Index: tests/toolkit/ezpinihelper.php =================================================================== --- tests/toolkit/ezpinihelper.php (revision 25359) +++ tests/toolkit/ezpinihelper.php (working copy) @@ -57,6 +57,19 @@ } /** + * Changes multiple INI settings values using setINISetting + * @param array $settings set of INI settings, as an array of 4 values + */ + public static function setINISettings( $settings ) + { + foreach( $settings as $iniSettings ) + { + list( $file, $block, $variable, $value ) = $iniSettings; + self::setINISetting( $file, $block, $variable, $value ); + } + } + + /** * Modified INI settings, as an array of 4 keys array: * file, block, variable, value * @var array Index: autoload/ezp_kernel.php =================================================================== --- autoload/ezp_kernel.php (revision 25359) +++ autoload/ezp_kernel.php (working copy) @@ -355,6 +355,7 @@ 'eZSOAPServer' => 'lib/ezsoap/classes/ezsoapserver.php', 'eZSSLZone' => 'kernel/classes/ezsslzone.php', 'eZScript' => 'kernel/classes/ezscript.php', + 'eZScriptClusterPurge' => 'kernel/private/classes/ezscriptclusterpurge.php', 'eZSearch' => 'kernel/classes/ezsearch.php', 'eZSearchEngine' => 'kernel/search/plugins/ezsearchengine/ezsearchengine.php', 'eZSearchFunctionCollection' => 'kernel/search/ezsearchfunctioncollection.php', Index: bin/php/clusterbinarypurge.php =================================================================== --- bin/php/clusterbinarypurge.php (revision 0) +++ bin/php/clusterbinarypurge.php (revision 0) @@ -0,0 +1,62 @@ + ( "eZ Publish binary files purge\n" . + "Physically purges expired binary files\n" . + "\n" . + "./bin/php/clusterbinarypurge.php" ), + 'use-session' => false, + 'use-modules' => false, + 'use-extensions' => true ) ); + +$script->startup(); + +$options = $script->getOptions( "[dry-run][iteration-sleep:][iteration-limit:][memory-monitoring]", +"", +array( 'dry-run' => 'Test mode, output the list of affected files without removing them', + 'iteration-sleep' => 'Amount of seconds to sleep between each iteration when performing a purge operation, can be a float. Default is one second.', + 'iteration-limit' => 'Amount of items to remove in each iteration when performing a purge operation. Default is 100.', + 'memory-monitoring' => 'If set, memory usage will be logged in var/log/clusterbinarypurge.log.' ) ); +$sys = eZSys::instance(); + +$script->initialize(); + +if ( !eZScriptClusterPurge::isRequired() ) +{ + $cli->error( "Your current cluster handler does not require binary purge" ); + $script->shutdown( 1 ); +} + +$purgeHandler = new eZScriptClusterPurge(); +if ( $options['dry-run'] ) +{ + $purgeHandler->optDryRun = true; +} + +if ( $options['iteration-sleep'] ) +{ + $purgeHandler->optIterationSleep = (int)( $options['iteration-sleep'] * 1000000 ); +} + +if ( $options['iteration-limit'] ) +{ + $purgeHandler->optIterationLimit = (int)$options['iteration-limit']; +} + +if ( $options['memory-monitoring'] ) +{ + $purgeHandler->opt = true; +} + +$purgeHandler->run(); + +$script->shutdown(); +?> Index: cronjobs/clusterbinarypurge.php =================================================================== --- cronjobs/clusterbinarypurge.php (revision 0) +++ cronjobs/clusterbinarypurge.php (revision 0) @@ -0,0 +1,17 @@ +error( "Your current cluster handler does not require binary purge" ); + $script->shutdown( 1 ); +} + +$purgeHandler = new eZScriptClusterPurge(); +$purgeHandler->run(); +?>