From b930abff26783002daa714de42fb2a677db5fb57 Mon Sep 17 00:00:00 2001 From: Daniel Neto Date: Fri, 10 Nov 2023 14:30:26 -0300 Subject: [PATCH] Update --- Dockerfile | 1 + docker-compose.yml | 2 + install/recreateCacheDB.php | 21 +-- objects/Object.php | 4 +- objects/playlist.php | 9 - plugin/Cache/Cache.php | 30 ++-- plugin/Cache/Objects/CacheDB.php | 75 +++++++++ plugin/Cache/Objects/CachesInDB.php | 33 ++-- plugin/Cache/Objects/CachesInDBMem.php | 223 +++++++++++++++++++++++++ plugin/Cache/install/memTable.sql | 28 ++++ plugin/Live/Live.php | 2 +- plugin/YPTSocket/MessageSQLite.php | 2 + plugin/YPTSocket/YPTSocket.php | 2 +- 13 files changed, 368 insertions(+), 64 deletions(-) create mode 100644 plugin/Cache/Objects/CacheDB.php create mode 100644 plugin/Cache/Objects/CachesInDBMem.php create mode 100644 plugin/Cache/install/memTable.sql diff --git a/Dockerfile b/Dockerfile index d98d3f5abc..ae745d9238 100644 --- a/Dockerfile +++ b/Dockerfile @@ -153,6 +153,7 @@ RUN dos2unix /usr/local/bin/docker-entrypoint && \ a2enmod rewrite expires headers ssl xsendfile RUN echo "error_log = /dev/stdout" >> /etc/php/8.1/apache2/php.ini +RUN echo "session.save_path = /var/www/memfolder" >> /etc/php/8.1/apache2/php.ini # Add Apache configuration RUN echo "ServerName localhost" >> /etc/apache2/apache2.conf diff --git a/docker-compose.yml b/docker-compose.yml index 39fe36b6cc..21d1ba4819 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,8 @@ services: WEBSITE_TITLE: ${WEBSITE_TITLE} MAIN_LANGUAGE: ${MAIN_LANGUAGE} restart: "unless-stopped" + tmpfs: + - /var/www/memfolder:rw,size=512M environment: SOCKET_PORT: ${SOCKET_PORT:-2053} HTTP_PORT: ${HTTP_PORT:-80} diff --git a/install/recreateCacheDB.php b/install/recreateCacheDB.php index 75c9989b82..d48ea7a957 100644 --- a/install/recreateCacheDB.php +++ b/install/recreateCacheDB.php @@ -15,25 +15,6 @@ error_reporting(E_ALL); $sql = 'DROP TABLE IF EXISTS `CachesInDB`'; $global['mysqli']->query($sql); $file = $global['systemRootPath'] . 'plugin/Cache/install/install.sql'; -$templine = ''; -$lines = file($file); - foreach ($lines as $line) { - if (substr($line, 0, 2) == '--' || $line == '') { - continue; - } - $templine .= $line; - if (substr(trim($line), -1, 1) == ';') { - try { - if (!$global['mysqli']->query($templine)) { - echo('Error performing query ' . $templine . ': ' . $global['mysqli']->error . PHP_EOL); - //exit; - } - } catch (Exception $exc) { - echo $exc->getTraceAsString(). PHP_EOL; - } - - $templine = ''; - } - } +sqlDal::executeFile($file); echo PHP_EOL . " Done! " . PHP_EOL; die(); diff --git a/objects/Object.php b/objects/Object.php index 78ddfb5659..24304638f7 100644 --- a/objects/Object.php +++ b/objects/Object.php @@ -496,7 +496,7 @@ abstract class ObjectYPT implements ObjectInterface return false; } if (class_exists('CachesInDB')) { - $content = CachesInDB::encodeContent($json); + $content = CacheDB::encodeContent($json); } else { $content = base64_encode($json); } @@ -1092,7 +1092,7 @@ abstract class CacheHandler { public function deleteCache() { $prefix = $this->getCacheSubdir(); if (class_exists('CachesInDB')) { - CachesInDB::_deleteCacheStartingWith($prefix); + CacheDB::deleteCacheStartingWith($prefix); } _session_start(); clearCache(true); diff --git a/objects/playlist.php b/objects/playlist.php index 9e0ca7d9c2..dd521c9b16 100644 --- a/objects/playlist.php +++ b/objects/playlist.php @@ -820,15 +820,6 @@ class PlayList extends ObjectYPT { static function deleteCacheDir($playlists_id) { $cacheHandler = new PlayListCacheHandler($playlists_id); $cacheHandler->deleteCache(); - /* - $tmpDir = ObjectYPT::getCacheDir(); - $name = "getvideosfromplaylist{$playlists_id}"; - $cacheDir = $tmpDir . $name . DIRECTORY_SEPARATOR; - rrmdir($cacheDir); - if (class_exists('CachesInDB')) { - CachesInDB::_deleteCacheStartingWith($name); - } - */ } public function delete() { diff --git a/plugin/Cache/Cache.php b/plugin/Cache/Cache.php index 5ed3636fde..f855e597ef 100644 --- a/plugin/Cache/Cache.php +++ b/plugin/Cache/Cache.php @@ -1,7 +1,7 @@ $domain, 'ishttps' => $ishttps, 'user_location' => $user_location, 'loggedType' => $loggedType]; @@ -332,7 +332,7 @@ class Cache extends PluginAbstract { return $cache_setCacheToSaveAtTheEnd[$name]; } $metadata = self::getCacheMetaData(); - return CachesInDB::_getCache($name, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType'], $ignoreMetadata); + return CacheDB::getCache($name, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType'], $ignoreMetadata); } public static function _setCache($name, $value) { @@ -342,17 +342,13 @@ class Cache extends PluginAbstract { } $cache_setCacheToSaveAtTheEnd[$name] = $value; return true; - //$metadata = self::getCacheMetaData(); - //return CachesInDB::_setCache($name, $value, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType']); } static function saveCache() { global $cache_setCacheToSaveAtTheEnd; if(!empty($cache_setCacheToSaveAtTheEnd)){ $metadata = self::getCacheMetaData(); - //mysqlBeginTransaction(); - CachesInDB::setBulkCache($cache_setCacheToSaveAtTheEnd, $metadata); - //mysqlCommit(); + CacheDB::setBulkCache($cache_setCacheToSaveAtTheEnd, $metadata); } } @@ -380,18 +376,12 @@ class Cache extends PluginAbstract { if (empty($_getCacheDB[$index])) { $_getCacheDB[$index] = null; $metadata = self::getCacheMetaData(); - $row = CachesInDB::_getCache($name, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType'], $ignoreMetadata); + $row = CacheDB::getCache($name, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType'], $ignoreMetadata); if (!empty($row)) { //$time = getTimeInTimezone(strtotime($row['modified']), $row['timezone']); $time = $row['created_php_time']; if (!empty($lifetime) && ($time + $lifetime) < time() && !empty($row['id'])) { $cacheNotFound++; - /* - $c = new CachesInDB($row['id']); - if (!empty($c->getId())) { - $c->delete(); - } - */ } else if(!empty($row['content'])) { $_getCacheDB[$index] = _json_decode($row['content']); if($_getCacheDB[$index] === null){ @@ -407,16 +397,16 @@ class Cache extends PluginAbstract { } public static function deleteCache($name) { - return CachesInDB::_deleteCache($name); + return CacheDB::deleteCache($name); } public static function deleteAllCache() { - return CachesInDB::_deleteAllCache(); + return CacheDB::deleteAllCache(); } public static function deleteFirstPageCache() { clearCache(true); - return CachesInDB::_deleteCacheStartingWith('firstPage'); + return CacheDB::deleteCacheStartingWith('firstPage'); } public static function deleteOldCache($days, $limit = 5000) { diff --git a/plugin/Cache/Objects/CacheDB.php b/plugin/Cache/Objects/CacheDB.php new file mode 100644 index 0000000000..c38ffe6733 --- /dev/null +++ b/plugin/Cache/Objects/CacheDB.php @@ -0,0 +1,75 @@ +errno == 1146){ + $error = array($global['mysqli']->error); + $file = $global['systemRootPath'] . 'plugin/Cache/install/install.sql'; + sqlDal::executeFile($file); + if (!static::isTableInstalled()) { + $error[] = $global['mysqli']->error; + die("We could not create table ".static::getTableName().'
'.implode('
', $error)); + } + } $row = false; } //var_dump($row); @@ -210,6 +218,7 @@ class CachesInDB extends ObjectYPT $values = []; $tz = date_default_timezone_get(); $time = time(); + $start = microtime(true); foreach ($cacheArray as $name => $cache) { $name = self::hashName($name); $content = !is_string($cache) ? _json_encode($cache) : $cache; @@ -230,7 +239,7 @@ class CachesInDB extends ObjectYPT array_push($values, $name, $content, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType'], $expires, $tz, $time); } - $sql = "INSERT INTO CachesInDB (name, content, domain, ishttps, user_location, loggedType, expires, timezone, created_php_time, created, modified) + $sql = "INSERT INTO " . static::getTableName() . " (name, content, domain, ishttps, user_location, loggedType, expires, timezone, created_php_time, created, modified) VALUES " . implode(", ", $placeholders) . " ON DUPLICATE KEY UPDATE content = VALUES(content), @@ -242,6 +251,8 @@ class CachesInDB extends ObjectYPT $result = sqlDAL::writeSql($sql, implode('', $formats), $values); //_error_log("setBulkCache writeSql total= ".count($placeholders)); //var_dump($result, $sql, implode('', $formats), $values);exit; + $end = number_format(microtime(true) - $start, 5); + _error_log("Disk setBulkCache took {$end} seconds"); return $result; } @@ -275,7 +286,7 @@ class CachesInDB extends ObjectYPT $name = self::hashName($name); //$sql = "DELETE FROM " . static::getTableName() . " "; //$sql .= " WHERE name LIKE '{$name}%'"; - $sql = "DELETE FROM CachesInDB WHERE MATCH(name) AGAINST('{$name}*' IN BOOLEAN MODE);"; + $sql = "DELETE FROM " . static::getTableName() . " WHERE MATCH(name) AGAINST('{$name}*' IN BOOLEAN MODE);"; $global['lastQuery'] = $sql; //_error_log("Delete Query: ".$sql); @@ -320,18 +331,18 @@ class CachesInDB extends ObjectYPT $content = _json_encode($content); } $prefix = substr($content, 0, 10); - if ($prefix!== CachesInDB::$prefix) { + if ($prefix!== CacheDB::$prefix) { //$content = base64_encode($content); - $content = CachesInDB::$prefix.$content; + $content = CacheDB::$prefix.$content; } return $content; } public static function decodeContent($content) { - $prefix = substr($content, 0, strlen(CachesInDB::$prefix)); - if ($prefix === CachesInDB::$prefix) { - $content = str_replace(CachesInDB::$prefix, '', $content); + $prefix = substr($content, 0, strlen(CacheDB::$prefix)); + if ($prefix === CacheDB::$prefix) { + $content = str_replace(CacheDB::$prefix, '', $content); //$content = base64_decode($content); } return $content; diff --git a/plugin/Cache/Objects/CachesInDBMem.php b/plugin/Cache/Objects/CachesInDBMem.php new file mode 100644 index 0000000000..c78a34bc9f --- /dev/null +++ b/plugin/Cache/Objects/CachesInDBMem.php @@ -0,0 +1,223 @@ +errno == 1146) { + self::tryToCreateTables(); + } + } + return false; + } + public static function setBulkCache($cacheArray, $metadata) + { + if (empty($cacheArray)) { + return false; + } + $start = microtime(true); + foreach ($cacheArray as $name => $cache) { + self::_setCache($name, $cache, $metadata['domain'], $metadata['ishttps'], $metadata['user_location'], $metadata['loggedType']); + } + + $end = number_format(microtime(true) - $start, 5); + _error_log("Memory setBulkCache took {$end} seconds"); + return true; + } + + + public static function _setCache($name, $content, $domain, $ishttps, $user_location, $loggedType) + { + if (!is_string($content)) { + $content = _json_encode($content); + } + + if (empty($content)){ + return false; + } + + global $global; + $time = time(); + $timezone = date_default_timezone_get();; + + // Preparing SQL for Metadata Insertion + $metadataSql = "INSERT INTO " . self::$metadataTable . " (name, domain, ishttps, user_location, loggedType, created, modified, expires, timezone, created_php_time) + VALUES (?, ?, ?, ?, ?, NOW(), NOW(), ?, ?, ?) + ON DUPLICATE KEY UPDATE + expires = VALUES(expires), + created_php_time = VALUES(created_php_time), + modified = NOW()"; + + $contentSqlBase = "INSERT INTO " . self::$contentTable . " (id, content) + VALUES (?, ?) + ON DUPLICATE KEY UPDATE + content = VALUES(content)"; + + $name = self::hashName($name); + + $expires = date('Y-m-d H:i:s', strtotime('+1 month')); + + $metadataSqlValues = [$name, $domain, $ishttps, $user_location, $loggedType, $expires, $timezone, $time]; + /** + * + * @var array $global + * @var object $global['mysqli'] + */ + + // Insert metadata + $metadataResult = sqlDAL::writeSql($metadataSql, 'sssssssi', $metadataSqlValues); + if ($metadataResult) { + $insertedId = @$global['mysqli']->insert_id; // Get the last inserted ID + // Insert content + if ($insertedId > 0) { + return sqlDAL::writeSql($contentSqlBase, 'is', [$insertedId, $content]); + } else { + return false; + } + } else { + return false; + } + } + + + public static function _deleteCache($name) + { + global $global; + if (empty($name)) { + return false; + } + + if (!static::isTableInstalled(self::$metadataTable)) { + return false; + } + $name = self::hashName($name); + + // Delete from content table + $sqlContent = "DELETE FROM " . self::$contentTable . " WHERE id IN (SELECT id FROM " . self::$metadataTable . " WHERE name = ?)"; + sqlDAL::writeSql($sqlContent, "s", [$name]); + + // Delete from metadata table + $sqlMetadata = "DELETE FROM " . self::$metadataTable . " WHERE name = ?"; + return sqlDAL::writeSql($sqlMetadata, "s", [$name]); + } + public static function _deleteCacheStartingWith($name) + { + global $global; + if (empty($name)) { + return false; + } + + if (!static::isTableInstalled(self::$metadataTable)) { + return false; + } + $name = self::hashName($name); + + // Delete from content table + $sqlContent = "DELETE FROM " . self::$contentTable . " WHERE id IN (SELECT id FROM " . self::$metadataTable . " WHERE name LIKE '{$name}%')"; + sqlDAL::writeSql($sqlContent); + + // Delete from metadata table + $sqlMetadata = "DELETE FROM " . self::$metadataTable . " WHERE name LIKE '{$name}%'"; + return sqlDAL::writeSql($sqlMetadata); + } + public static function _deleteCacheWith($name) + { + global $global; + if (empty($name)) { + return false; + } + + if (!static::isTableInstalled(self::$metadataTable)) { + return false; + } + $name = self::hashName($name); + + // Delete from content table + $sqlContent = "DELETE FROM " . self::$contentTable . " WHERE id IN (SELECT id FROM " . self::$metadataTable . " WHERE name LIKE '%{$name}%')"; + sqlDAL::writeSql($sqlContent); + + // Delete from metadata table + $sqlMetadata = "DELETE FROM " . self::$metadataTable . " WHERE name LIKE '%{$name}%'"; + return sqlDAL::writeSql($sqlMetadata); + } + public static function _deleteAllCache() + { + global $global; + + if (!static::isTableInstalled(self::$metadataTable)) { + return false; + } + + // Truncate content table + $sqlContent = "TRUNCATE TABLE " . self::$contentTable . ""; + sqlDAL::writeSql($sqlContent); + + // Truncate metadata table + $sqlMetadata = "TRUNCATE TABLE " . self::$metadataTable . ""; + return sqlDAL::writeSql($sqlMetadata); + } +} diff --git a/plugin/Cache/install/memTable.sql b/plugin/Cache/install/memTable.sql new file mode 100644 index 0000000000..77fda97f16 --- /dev/null +++ b/plugin/Cache/install/memTable.sql @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS `CachesInDB_Memory`; +DROP TABLE IF EXISTS `CachesInDB_Blob`; + +CREATE TABLE IF NOT EXISTS `CachesInDB_Memory` ( + `id` INT NOT NULL AUTO_INCREMENT, + `created` DATETIME NULL, + `modified` DATETIME NULL, + `domain` VARCHAR(100) NULL, + `ishttps` TINYINT NULL, + `loggedType` ENUM('n', 'l', 'a') NULL DEFAULT 'n' COMMENT 'n=not logged\nl=logged\na=admin', + `user_location` VARCHAR(100) NULL, + `expires` DATETIME NULL, + `timezone` VARCHAR(100) NULL, + `created_php_time` INT(11) NULL, + `name` VARCHAR(500) NULL, + PRIMARY KEY (`id`), + INDEX `caches1` (`domain` ASC), + INDEX `caches2` (`ishttps` ASC), + INDEX `caches3` (`loggedType` ASC), + INDEX `caches4` (`user_location` ASC), + INDEX `caches9` (`name` ASC)) +ENGINE = MEMORY; + +CREATE TABLE IF NOT EXISTS `CachesInDB_Blob` ( + `id` INT NOT NULL, + `content` MEDIUMBLOB NULL, + PRIMARY KEY (`id`) +) ENGINE = MyISAM; diff --git a/plugin/Live/Live.php b/plugin/Live/Live.php index 627361fd21..fe0dc726d3 100644 --- a/plugin/Live/Live.php +++ b/plugin/Live/Live.php @@ -3067,7 +3067,7 @@ Click here to join our live."; _error_log("deleteStatsCache: {$cacheDir} " . json_encode(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS))); rrmdir($cacheDir); if(class_exists('CachesInDB')){ - CachesInDB::_deleteCacheWith('getStats'); + CacheDB::deleteCacheWith('getStats'); } if ($clearFirstPage) { clearCache(true); diff --git a/plugin/YPTSocket/MessageSQLite.php b/plugin/YPTSocket/MessageSQLite.php index 0ca2789c1d..1d540bed94 100644 --- a/plugin/YPTSocket/MessageSQLite.php +++ b/plugin/YPTSocket/MessageSQLite.php @@ -468,11 +468,13 @@ class Message implements MessageComponentInterface { if($client['time']+$this->disconnectAfter < $time){ _error_log("resourceId={$client['resourceId']} is too old, close it"); $this->clients[$client['resourceId']]->close(); + unset($this->clients[$client['resourceId']]); } if($client['isCommandLine']){ if($client['time']+60 < $time){ _error_log("resourceId={$client['resourceId']} disconnect commandline after 1 min"); $this->clients[$client['resourceId']]->close(); + unset($this->clients[$client['resourceId']]); } //_error_log("msgToAll continue"); continue; diff --git a/plugin/YPTSocket/YPTSocket.php b/plugin/YPTSocket/YPTSocket.php index 808fc22d22..92ecbc07bc 100644 --- a/plugin/YPTSocket/YPTSocket.php +++ b/plugin/YPTSocket/YPTSocket.php @@ -41,7 +41,7 @@ class YPTSocket extends PluginAbstract { } public static function getServerVersion() { - return "5.1"; + return "5.2"; } public function updateScript() {