diff -urN autoload/ezp_kernel.php autoload/ezp_kernel.php --- autoload/ezp_kernel.php 2009-09-30 12:22:30.000000000 +0200 +++ autoload/ezp_kernel.php 2011-04-29 10:28:04.630371717 +0200 @@ -355,6 +355,7 @@ 'eZSOAPServer' => 'lib/ezsoap/classes/ezsoapserver.php', 'eZSSLZone' => 'kernel/classes/ezsslzone.php', 'eZScript' => 'kernel/classes/ezscript.php', + 'eZScriptClusterPurge' => 'kernel/private/classes/ezscriptclusterpurge.php', 'eZSearch' => 'kernel/classes/ezsearch.php', 'eZSearchEngine' => 'kernel/search/plugins/ezsearchengine/ezsearchengine.php', 'eZSearchFunctionCollection' => 'kernel/search/ezsearchfunctioncollection.php', diff -urN bin/php/clusterpurge.php bin/php/clusterpurge.php --- bin/php/clusterpurge.php 1970-01-01 01:00:00.000000000 +0100 +++ bin/php/clusterpurge.php 2011-04-29 10:40:06.462373318 +0200 @@ -0,0 +1,75 @@ + ( "eZ Publish cluster files purge\n" . + "Physically purges files\n" . + "\n" . + "./bin/php/clusterpurge.php --scopes=scope1,scope2" ), + 'use-session' => false, + 'use-modules' => false, + 'use-extensions' => true ) ); + +$script->startup(); + +$options = $script->getOptions( "[dry-run][iteration-sleep:][iteration-limit:][memory-monitoring][scopes:][expiry:]", +"", +array( 'dry-run' => 'Test mode, output the list of affected files without removing them', + 'iteration-sleep' => 'Amount of seconds to sleep between each iteration when performing a purge operation, can be a float. Default is one second.', + 'iteration-limit' => 'Amount of items to remove in each iteration when performing a purge operation. Default is 100.', + 'memory-monitoring' => 'If set, memory usage will be logged in var/log/clusterpurge.log.', + 'scopes' => 'Comma separated list of file types to purge. Possible values are: classattridentifiers, classidentifiers, content, expirycache, statelimitations, template-block, user-info-cache, viewcache, wildcard-cache-index, image, binaryfile', + 'expiry' => 'Number of days since the file was expired. Only files older than this will be purged. Default is 30, minimum is 1.' ) ); +$sys = eZSys::instance(); + +$script->initialize(); + +if ( !eZScriptClusterPurge::isRequired() ) +{ + $cli->error( "Your current cluster handler does not require files purge" ); + $script->shutdown( 1 ); +} + +$purgeHandler = new eZScriptClusterPurge(); +if ( $options['dry-run'] ) +{ + $purgeHandler->optDryRun = true; +} + +if ( $options['iteration-sleep'] ) +{ + $purgeHandler->optIterationSleep = (int)( $options['iteration-sleep'] * 1000000 ); +} + +if ( $options['iteration-limit'] ) +{ + $purgeHandler->optIterationLimit = (int)$options['iteration-limit']; +} + +if ( $options['memory-monitoring'] ) +{ + $purgeHandler->optMemoryMonitoring = true; +} + +if ( $options['scopes'] ) +{ + $purgeHandler->optScopes = explode( ',', $options['scopes'] ); +} + +if ( $options['expiry'] ) +{ + $purgeHandler->optExpiry = (int)$options['expiry'] * 86400; // 60*60*24 +} + +$purgeHandler->run(); + +$script->shutdown(); + +?> diff -urN cronjobs/clusterpurge.php cronjobs/clusterpurge.php --- cronjobs/clusterpurge.php 1970-01-01 01:00:00.000000000 +0100 +++ cronjobs/clusterpurge.php 2011-04-29 10:40:06.462373318 +0200 @@ -0,0 +1,30 @@ +error( "Your current cluster handler does not require file purge" ); + $script->shutdown( 1 ); +} + +$purgeHandler = new eZScriptClusterPurge(); +$purgeHandler->optScopes = array( 'classattridentifiers', + 'classidentifiers', + 'content', + 'expirycache', + 'statelimitations', + 'template-block', + 'user-info-cache', + 'viewcache', + 'wildcard-cache-index', + 'image', + 'binaryfile' ); +$purgeHandler->optExpiry = 30; +$purgeHandler->run(); + +?> diff -urN kernel/classes/clusterfilehandlers/dbbackends/mysql.php kernel/classes/clusterfilehandlers/dbbackends/mysql.php --- kernel/classes/clusterfilehandlers/dbbackends/mysql.php 2009-09-30 12:22:47.000000000 +0200 +++ kernel/classes/clusterfilehandlers/dbbackends/mysql.php 2011-04-29 10:40:06.464373250 +0200 @@ -1662,6 +1662,40 @@ return ( $row[0] + $this->dbparams['cache_generation_timeout'] ) - time(); } + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * @param int $expiry Number of seconds, only items older than this will be returned. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = false ) + { + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + $query = "SELECT name FROM " . TABLE_METADATA . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $expiry !== false ) + { + $query .= ' AND mtime < ' . (time() - $expiry); + } + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[0]}, {$limit[1]}"; + } + $res = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = mysql_fetch_row( $res ) ) + $filePathList[] = $row[0]; + + return $filePathList; + } + public $db = null; public $numQueries = 0; public $transactionCount = 0; diff -urN kernel/classes/clusterfilehandlers/ezdbfilehandler.php kernel/classes/clusterfilehandlers/ezdbfilehandler.php --- kernel/classes/clusterfilehandlers/ezdbfilehandler.php 2009-09-30 12:22:47.000000000 +0200 +++ kernel/classes/clusterfilehandlers/ezdbfilehandler.php 2011-04-29 10:40:06.465373460 +0200 @@ -1306,6 +1306,62 @@ } /** + * eZDB does require binary purge. + * It does store files in DB and therefore doesn't remove files in real time + * + * @since 4.3.0 + * @deprecated Deprecated as of 4.5, use {@link eZDBFileHandler::requiresPurge()} instead. + * @return bool + */ + public function requiresBinaryPurge() + { + return true; + } + + /** + * eZDB does require binary purge. + * It does store files in DB and therefore doesn't remove files in real time + * + * @since 4.5.0 + * @return bool + **/ + public function requiresPurge() + { + return true; + } + + /** + * Fetches the first $limit expired binary items from the DB + * + * @param array $limit A 2 items array( offset, limit ) + * + * @return array(eZClusterFileHandlerInterace) + * @since 4.3.0 + * @deprecated Deprecated as of 4.5, use {@link eZDBFileHandler::fetchExpiredItems()} instead. + * + * @todo handle output using $cli or something + */ + public function fetchExpiredBinaryItems( $limit = array( 0, 100 ) ) + { + return $this->backend->fetchExpiredItems( array( 'image', 'binaryfile' ), $limit ); + } + + /** + * Fetches the first $limit expired files from the DB + * + * @param array $scopes Array of scopes to fetch from + * @param array $limit A 2 items array( offset, limit ) + * @param int $expiry Number of seconds, only items older than this will be returned + * + * @return array(filepath) + * @since 4.5.0 + */ + public function fetchExpiredItems( $scopes, $limit = array( 0 , 100 ), $expiry = false ) + { + return $this->backend->expiredFilesList( $scopes, $limit, $expiry ); + } + + /** * Database backend class * @var eZDBFileHandlerMysqlBackend **/ diff -urN kernel/classes/clusterfilehandlers/ezfsfilehandler.php kernel/classes/clusterfilehandlers/ezfsfilehandler.php --- kernel/classes/clusterfilehandlers/ezfsfilehandler.php 2009-09-30 12:22:47.000000000 +0200 +++ kernel/classes/clusterfilehandlers/ezfsfilehandler.php 2011-04-29 10:40:06.467371018 +0200 @@ -1048,6 +1048,31 @@ return false; } + /** + * eZFS does not require binary purge. + * Files are stored on plain FS and removed using FS functions + * + * @since 4.3 + * @deprecated Deprecated as of 4.5, use {@link eZFSFileHandler::requiresPurge()} instead. + * @return bool + */ + public function requiresBinaryPurge() + { + return false; + } + + /** + * eZFS does not require binary purge. + * Files are stored on plain FS and removed using FS functions + * + * @since 4.5.0 + * @return bool + **/ + public function requiresPurge() + { + return false; + } + public $metaData = null; public $filePath; } diff -urN kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php --- kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php 2009-09-30 12:22:57.000000000 +0200 +++ kernel/private/classes/clusterfilehandlers/dfsbackends/mysql.php 2011-04-29 10:40:06.467371018 +0200 @@ -1645,6 +1645,40 @@ } /** + * Returns the list of expired binary files (images + binaries) + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * @param int $expiry Number of seconds, only items older than this will be returned. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = false ) + { + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + $query = "SELECT name FROM " . self::TABLE_METADATA . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $expiry !== false ) + { + $query .= ' AND mtime < ' . (time() - $expiry); + } + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[0]}, {$limit[1]}"; + } + $res = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = mysql_fetch_row( $res ) ) + $filePathList[] = $row[0]; + + return $filePathList; + } + + /** * DB connexion handle * @var handle **/ diff -urN kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php --- kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php 2009-09-30 12:22:57.000000000 +0200 +++ kernel/private/classes/clusterfilehandlers/ezdfsfilehandler.php 2011-04-29 10:40:06.467371018 +0200 @@ -1398,6 +1398,62 @@ } /** + * eZDFS does require binary purge. + * It does store files in DB + on NFS, and therefore doesn't remove files + * in real time + * + * @since 4.3 + * @deprecated Deprecated as of 4.5, use {@link eZDFSFileHandler::requiresPurge()} instead. + * @return bool + */ + public function requiresBinaryPurge() + { + return true; + } + + /** + * eZDFS does require binary purge. + * It does store files in DB + on NFS, and therefore doesn't remove files + * in real time + * + * @since 4.5.0 + * @return bool + **/ + public function requiresPurge() + { + return true; + } + + /** + * Fetches the first $limit expired binary items from the DB + * + * @param array $limit A 2 items array( offset, limit ) + * + * @return array(filepath) + * @since 4.3.0 + * @deprecated Deprecated as of 4.5, use {@link eZDFSFileHandler::fetchExpiredItems()} instead. + */ + public function fetchExpiredBinaryItems( $limit = array( 0 , 100 ) ) + { + return self::$dbbackend->fetchExpiredItems( array( 'image', 'binaryfile' ), $limit ); + } + + /** + * Fetches the first $limit expired files from the DB + * + * @param array $scopes Array of scopes to fetch from + * @param array $limit A 2 items array( offset, limit ) + * @param int $expiry Number of seconds, only items older than this will be returned + * + * @return array(filepath) + * @since 4.5.0 + */ + public function fetchExpiredItems( $scopes, $limit = array( 0 , 100 ), $expiry = false ) + { + return self::$dbbackend->expiredFilesList( $scopes, $limit, $expiry ); + } + + /** * Database backend class * Provides metadata operations * @var eZDFSFileHandlerMySQLBackend diff -urN kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php --- kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php 2009-09-30 12:22:57.000000000 +0200 +++ kernel/private/classes/clusterfilehandlers/ezfs2filehandler.php 2011-04-29 10:40:06.468371366 +0200 @@ -771,6 +771,31 @@ } /** + * eZFS2 doesn't require purge as it already purges files in realtime + * (FS based) + * + * @since 4.3 + * @deprecated Deprecated as of 4.5, use {@link eZFS2FileHandler::requiresPurge()} instead. + * @return bool + */ + public function requiresBinaryPurge() + { + return false; + } + + /** + * eZFS2 doesn't require purge as it already purges files in realtime + * (FS based) + * + * @since 4.5.0 + * @return bool + **/ + public function requiresPurge() + { + return false; + } + + /** * holds the real file path. This is only used when we are generating a cache * file, in which case $filePath holds the generating cache file name, * and $realFilePath holds the real name diff -urN kernel/private/classes/ezscriptclusterpurge.php kernel/private/classes/ezscriptclusterpurge.php --- kernel/private/classes/ezscriptclusterpurge.php 1970-01-01 01:00:00.000000000 +0100 +++ kernel/private/classes/ezscriptclusterpurge.php 2011-04-29 10:40:06.468371366 +0200 @@ -0,0 +1,198 @@ +options = array( + 'dry-run' => false, + 'iteration-sleep' => 1, + 'iteration-limit' => 100, + 'memory-monitoring' => false, + 'scopes' => false, + 'expiry' => 2592000 // 60*60*24*30 = 30 days + ); + } + + /** + * Performs preliminary checks in order to ensure the process can be + * started: + * - does the active cluster handler require purging of binary files + * + * @return bool + */ + public static function isRequired() + { + $clusterHandler = eZClusterFileHandler::instance(); + $result = $clusterHandler->requiresPurge(); + + return $result; + } + + /** + * Executes the purge operation + * + * @todo Endless loop on fetch list. The expired items are returned over and over again + **/ + public function run() + { + $cli = eZCLI::instance(); + + if ( $this->optMemoryMonitoring == true ) + { + eZLog::rotateLog( self::LOG_FILE ); + $cli->notice( "Logging memory usage to " . self::LOG_FILE ); + } + + if ( $this->optIterationSleep > 0 ) + $sleep = ( $this->optIterationSleep * 1000000 ); + else + $sleep = false; + + $limit = array( 0, $this->optIterationLimit ); + + $cli->output( "Purging expired items:" ); + + self::monitor( "start" ); + + // Fetch a limited list of purge items from the handler itself + $clusterHandler = eZClusterFileHandler::instance(); + while ( $filesList = $clusterHandler->fetchExpiredItems( $this->optScopes, $limit, $this->optExpiry ) ) + { + self::monitor( "iteration start" ); + foreach( $filesList as $file ) + { + $cli->output( "- $file" ); + if ( $this->optDryRun == false ) + { + self::monitor( "purge" ); + $fh = eZClusterFileHandler::instance( $file ); + $fh->purge( false, false ); + unset( $fh ); + } + } + if ( $sleep !== false ) + usleep( $sleep ); + + // the offset only has to be increased in dry run mode + // since each batch is not deleted + if ( $this->optDryRun == true ) + { + $limit[0] += $limit[1]; + } + self::monitor( "iteration end" ); + } + + self::monitor( "end" ); + } + + public function __get( $propertyName ) + { + switch( $propertyName ) + { + case 'optDryRun': + { + return $this->options['dry-run']; + } break; + + // no sleep in dry-run, it's not nap time ! + case 'optIterationSleep': + { + if ( $this->optDryRun == true ) + return 0; + else + return $this->options['iteration-sleep']; + } break; + + case 'optIterationLimit': + { + return $this->options['iteration-limit']; + } break; + + case 'optMemoryMonitoring': + { + return $this->options['memory-monitoring']; + } break; + + case 'optScopes': + { + return $this->options['scopes']; + } break; + + case 'optExpiry': + { + return $this->options['expiry']; + } break; + } + } + + /** + * @todo Add type & value check + */ + public function __set( $propertyName, $propertyValue ) + { + switch( $propertyName ) + { + case 'optDryRun': + { + $this->options['dry-run'] = $propertyValue; + } break; + + case 'optIterationSleep': + { + return $this->options['iteration-sleep'] = $propertyValue; + } break; + + case 'optIterationLimit': + { + $this->options['iteration-limit'] = $propertyValue; + } break; + + case 'optMemoryMonitoring': + { + $this->options['memory-monitoring'] = $propertyValue; + } break; + + case 'optScopes': + { + $this->options['scopes'] = $propertyValue; + } break; + + case 'optExpiry': + { + $this->options['expiry'] = $propertyValue; + } break; + } + } + + public function monitor( $text ) + { + if ( $this->opt == true ) + { + eZLog::write( "mem [$text]: " . memory_get_usage(), self::LOG_FILE ); + } + } + + private $options = array(); + + const LOG_FILE = 'clusterpurge.log'; +} +?> diff -urN kernel/private/interfaces/ezclusterfilehandlerinterface.php kernel/private/interfaces/ezclusterfilehandlerinterface.php --- kernel/private/interfaces/ezclusterfilehandlerinterface.php 2009-09-30 12:22:58.000000000 +0200 +++ kernel/private/interfaces/ezclusterfilehandlerinterface.php 2011-04-29 10:40:06.468371366 +0200 @@ -390,5 +390,24 @@ * @return bool **/ public function requiresClusterizing(); + + /** + * This method indicates if the cluster file handler requires binary files + * to be purged in order to be physically deleted + * + * @since 4.3 + * @deprecated Deprecated as of 4.5, use {@link eZClusterFileHandlerInterface::requiresPurge()} instead. + * @return bool + */ + public function requiresBinaryPurge(); + + /** + * This method indicates if the cluster file handler requires binary files + * to be purged in order to be physically deleted + * + * @since 4.5 + * @return bool + */ + public function requiresPurge(); } ?> diff -urN settings/cronjob.ini settings/cronjob.ini --- settings/cronjob.ini 2009-09-30 12:18:59.000000000 +0200 +++ settings/cronjob.ini 2011-04-29 10:40:06.468371366 +0200 @@ -43,6 +43,9 @@ [CronjobPart-unlock] Scripts[]=unlock.php +[CronjobPart-cluster_maintenance] +Scripts[]=clusterpurge.php + # Example of a cronjob part # This one will only run the workflow cronjob script #