diff --git a/composer.json b/composer.json index 9542fc5a1..3bc3e3675 100644 --- a/composer.json +++ b/composer.json @@ -61,7 +61,7 @@ "php artisan package:discover" ], "post-update-cmd": [ - "php artisan october:util set build", + "php artisan october:version", "php artisan package:discover" ], "test": [ diff --git a/modules/system/ServiceProvider.php b/modules/system/ServiceProvider.php index eb5b6c613..239465b46 100644 --- a/modules/system/ServiceProvider.php +++ b/modules/system/ServiceProvider.php @@ -133,7 +133,7 @@ class ServiceProvider extends ModuleServiceProvider protected function registerPrivilegedActions() { $requests = ['/combine/', '@/system/updates', '@/system/install', '@/backend/auth']; - $commands = ['october:up', 'october:update']; + $commands = ['october:up', 'october:update', 'october:version']; /* * Requests @@ -248,6 +248,8 @@ class ServiceProvider extends ModuleServiceProvider $this->registerConsoleCommand('october.env', 'System\Console\OctoberEnv'); $this->registerConsoleCommand('october.install', 'System\Console\OctoberInstall'); $this->registerConsoleCommand('october.passwd', 'System\Console\OctoberPasswd'); + $this->registerConsoleCommand('october.version', 'System\Console\OctoberVersion'); + $this->registerConsoleCommand('october.manifest', 'System\Console\OctoberManifest'); $this->registerConsoleCommand('plugin.install', 'System\Console\PluginInstall'); $this->registerConsoleCommand('plugin.remove', 'System\Console\PluginRemove'); diff --git a/modules/system/classes/FileManifest.php b/modules/system/classes/FileManifest.php new file mode 100644 index 000000000..a54b50ea3 --- /dev/null +++ b/modules/system/classes/FileManifest.php @@ -0,0 +1,193 @@ +setRoot($root); + } else { + $this->setRoot(base_path()); + } + + if (isset($modules)) { + $this->setModules($modules); + } else { + $this->setModules(Config::get('cms.loadModules', ['System', 'Backend', 'Cms'])); + } + } + + /** + * Sets the root folder. + * + * @param string $root + * @throws ApplicationException If the specified root does not exist. + */ + public function setRoot($root) + { + if (is_string($root)) { + $this->root = realpath($root); + + if ($this->root === false || !is_dir($this->root)) { + throw new ApplicationException( + 'Invalid root specified for the file manifest.' + ); + } + } + + return $this; + } + + /** + * Sets the modules. + * + * @param array $modules + */ + public function setModules(array $modules) + { + $this->modules = array_map(function ($module) { + return strtolower($module); + }, $modules); + + return $this; + } + + /** + * Gets a list of files and their corresponding hashsums. + * + * @return array + */ + public function getFiles() + { + if (count($this->files)) { + return $this->files; + } + + $files = []; + + foreach ($this->modules as $module) { + $path = $this->root . '/modules/' . $module; + + if (!is_dir($path)) { + continue; + } + + foreach ($this->findFiles($path) as $file) { + $files[$this->getFilename($file)] = hash('sha3-256', $this->normalizeFileContents($file)); + } + } + + return $this->files = $files; + } + + /** + * Gets the checksum of a specific install. + * + * @return array + */ + public function getModuleChecksums() + { + if (!count($this->files)) { + $this->getFiles(); + } + + $modules = []; + foreach ($this->modules as $module) { + $modules[$module] = ''; + } + + foreach ($this->files as $path => $hash) { + // Determine module + $module = explode('/', $path)[2]; + + $modules[$module] .= $hash; + } + + return array_map(function ($moduleSum) { + return hash('sha3-256', $moduleSum); + }, $modules); + } + + /** + * Finds all files within the path. + * + * @param string $basePath The base path to look for files within. + * @return array + */ + protected function findFiles(string $basePath) + { + $datasource = new FileDatasource($basePath, new Filesystem); + + $files = array_map(function ($path) use ($basePath) { + return $basePath . '/' . $path; + }, array_keys($datasource->getAvailablePaths())); + + // Ensure files are sorted so they are in a consistent order, no matter the way the OS returns the file list. + sort($files, SORT_NATURAL); + + return $files; + } + + /** + * Returns the filename without the root. + * + * @param string $file + * @return string + */ + protected function getFilename(string $file): string + { + return str_replace($this->root, '', $file); + } + + /** + * Normalises the file contents, irrespective of OS. + * + * @param string $file + * @return string + */ + protected function normalizeFileContents(string $file): string + { + if (!is_file($file)) { + return ''; + } + + $contents = file_get_contents($file); + + return str_replace(PHP_EOL, "\n", $contents); + } +} diff --git a/modules/system/classes/SourceManifest.php b/modules/system/classes/SourceManifest.php new file mode 100644 index 000000000..9e154e078 --- /dev/null +++ b/modules/system/classes/SourceManifest.php @@ -0,0 +1,356 @@ +setSource($source); + } else { + $this->setSource( + Config::get( + 'cms.sourceManifestUrl', + 'https://raw.githubusercontent.com/octoberrain/meta/master/manifest/builds.json' + ) + ); + } + + if ($autoload) { + $this->load(); + } + } + + /** + * Sets the source manifest URL. + * + * @param string $source + * @return void + */ + public function setSource($source) + { + if (is_string($source)) { + $this->source = $source; + } + } + + /** + * Loads the manifest file. + * + * @throws ApplicationException If the manifest is invalid, or cannot be parsed. + */ + public function load() + { + $source = file_get_contents($this->source); + if (empty($source)) { + throw new ApplicationException( + 'Source manifest not found' + ); + } + + $data = json_decode($source, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + throw new ApplicationException( + 'Unable to decode source manifest JSON data. JSON Error: ' . json_last_error_msg() + ); + } + if (!isset($data['manifest']) || !is_array($data['manifest'])) { + throw new ApplicationException( + 'The source manifest at "' . $this->source . '" does not appear to be a valid source manifest file.' + ); + } + + foreach ($data['manifest'] as $build) { + $this->builds[$build['build']] = [ + 'modules' => $build['modules'], + 'files' => $build['files'], + ]; + } + + return $this; + } + + /** + * Adds a FileManifest instance as a build to this source manifest. + * + * Changes between builds are calculated and stored with the build. Builds are stored numerically, in ascending + * order. + * + * @param integer $build Build number. + * @param FileManifest $manifest The file manifest to add as a build. + * @param integer $previous The previous build number, used to determine changes with this build. + * @return void + */ + public function addBuild($build, FileManifest $manifest, $previous = null) + { + $this->builds[(int) $build] = [ + 'modules' => $manifest->getModuleChecksums(), + 'files' => $this->processChanges($manifest, $previous), + ]; + + // Sort builds numerically in ascending order. + ksort($this->builds[$build], SORT_NUMERIC); + } + + /** + * Gets all builds. + * + * @return array + */ + public function getBuilds() + { + return $this->builds; + } + + /** + * Gets the maximum build number in the manifest. + * + * @return int + */ + public function getMaxBuild() + { + if (!count($this->builds)) { + return null; + } + + return max(array_keys($this->builds)); + } + + /** + * Generates the JSON data to be stored with the source manifest. + * + * @throws ApplicationException If no builds have been added to this source manifest. + * @return string + */ + public function generate() + { + if (!count($this->builds)) { + throw new ApplicationException( + 'No builds have been added to the manifest.' + ); + } + + $json = [ + 'manifest' => [], + ]; + + foreach ($this->builds as $build => $details) { + $json['manifest'][] = [ + 'build' => $build, + 'modules' => $details['modules'], + 'files' => $details['files'], + ]; + } + + return json_encode($json, JSON_PRETTY_PRINT); + } + + /** + * Gets the filelist state at a selected build. + * + * This method will list all expected files and hashsums at the specified build number. + * + * @param integer $build Build number to get the filelist state for. + * @throws ApplicationException If the specified build has not been added to the source manifest. + * @return array + */ + public function getState($build) + { + if (!isset($this->builds[$build])) { + throw new \Exception('The specified build has not been added.'); + } + + $state = []; + + foreach ($this->builds as $number => $details) { + if (isset($details['files']['added'])) { + foreach ($details['files']['added'] as $filename => $sum) { + $state[$filename] = $sum; + } + } + if (isset($details['files']['modified'])) { + foreach ($details['files']['modified'] as $filename => $sum) { + $state[$filename] = $sum; + } + } + if (isset($details['files']['removed'])) { + foreach ($details['files']['removed'] as $filename) { + unset($state[$filename]); + } + } + + if ($number === $build) { + break; + } + } + + return $state; + } + + /** + * Compares a file manifest with the source manifest. + * + * This will determine the build of the October CMS installation. + * + * @param FileManifest $manifest The file manifest to compare against the source. + * @return array|null Will return an array with the build, modified state and the probability that it is the + * version specified. If the detected version does not look like a likely candidate, this will return null. + */ + public function compare(FileManifest $manifest) + { + $modules = $manifest->getModuleChecksums(); + + // Look for an unmodified version + foreach ($this->getBuilds() as $build => $details) { + $matched = array_intersect_assoc($details['modules'], $modules); + + if (count($matched) === count($modules)) { + return [ + 'build' => $build, + 'modified' => false, + ]; + } + } + + // If we could not find an unmodified version, try to find the closest version and assume this is a modified + // install. + $buildMatch = []; + + foreach ($this->getBuilds() as $build => $details) { + $state = $this->getState($build); + + // Include only the files that match the modules being loaded in this file manifest + $availableModules = array_keys($modules); + + foreach ($state as $file => $sum) { + // Determine module + $module = explode('/', $file)[2]; + + if (!in_array($module, $availableModules)) { + unset($state[$file]); + } + } + + $filesExpected = count($state); + $filesFound = []; + $filesChanged = []; + + foreach ($manifest->getFiles() as $file => $sum) { + // Unknown new file + if (!isset($state[$file])) { + $filesChanged[] = $file; + continue; + } + + // Modified file + if ($state[$file] !== $sum) { + $filesFound[] = $file; + $filesChanged[] = $file; + continue; + } + + // Pristine file + $filesFound[] = $file; + } + + $foundPercent = count($filesFound) / $filesExpected; + $changedPercent = count($filesChanged) / $filesExpected; + + $score = ((1 * $foundPercent) - $changedPercent); + $buildMatch[$build] = round($score * 100, 2); + } + + + // Find likely version (we have to be at least 60% sure) + $likelyBuild = array_search(max($buildMatch), $buildMatch); + if ($buildMatch[$likelyBuild] < 60) { + return null; + } + + return [ + 'build' => $likelyBuild, + 'modified' => true, + ]; + } + + /** + * Determines file changes between the specified build and the previous build. + * + * Will return an array of added, modified and removed files. + * + * @param FileManifest $manifest The current build's file manifest. + * @param integer $previous The previous build number, used to determine changes with this build. + * @return array + */ + protected function processChanges(FileManifest $manifest, $previous = null) + { + // If no previous build has been provided, all files are added + if (is_null($previous)) { + return [ + 'added' => $manifest->getFiles(), + ]; + } + + // Only save files if they are changing the "state" of the manifest (ie. the file is modified, added or removed) + $state = $this->getState($previous); + $added = []; + $modified = []; + + foreach ($manifest->getFiles() as $file => $sum) { + if (!isset($state[$file])) { + $added[$file] = $sum; + continue; + } else { + if ($state[$file] !== $sum) { + $modified[$file] = $sum; + } + unset($state[$file]); + } + } + + // Any files still left in state have been removed + $removed = array_keys($state); + + $changes = []; + if (count($added)) { + $changes['added'] = $added; + } + if (count($modified)) { + $changes['modified'] = $modified; + } + if (count($removed)) { + $changes['removed'] = $removed; + } + + return $changes; + } +} diff --git a/modules/system/classes/UpdateManager.php b/modules/system/classes/UpdateManager.php index 1b9c3618f..075bdafdc 100644 --- a/modules/system/classes/UpdateManager.php +++ b/modules/system/classes/UpdateManager.php @@ -358,23 +358,38 @@ class UpdateManager } /** - * Asks the gateway for the lastest build number and stores it. + * Determines build number from source manifest. + * + * An array will be returned with the following, if a build can be determined: + * - `build` - The detected build number installed. + * - `modified` - Whether the installation appears to have been modified. + * + * Otherwise, `null` will be returned. + * + * @return array|null + */ + public function getBuildNumberManually() + { + $source = new SourceManifest(); + $manifest = new FileManifest(null, null, true); + + // Find build by comparing with source manifest + return $source->compare($manifest); + } + + /** + * Sets the build number in the database. + * * @return void */ public function setBuildNumberManually() { - $postData = []; + $build = $this->getBuildNumberManually(); - if (Config::get('cms.edgeUpdates', false)) { - $postData['edge'] = 1; + if (!is_null($build)) { + $this->setBuild($build['build'], null, $build['modified']); } - $result = $this->requestServerData('ping', $postData); - - $build = (int) array_get($result, 'pong', 420); - - $this->setBuild($build); - return $build; } @@ -457,12 +472,14 @@ class UpdateManager * Sets the build number and hash * @param string $hash * @param string $build + * @param bool $modified * @return void */ - public function setBuild($build, $hash = null) + public function setBuild($build, $hash = null, $modified = false) { $params = [ - 'system::core.build' => $build + 'system::core.build' => $build, + 'system::core.modified' => $modified, ]; if ($hash) { diff --git a/modules/system/console/OctoberManifest.php b/modules/system/console/OctoberManifest.php new file mode 100644 index 000000000..b92db74a7 --- /dev/null +++ b/modules/system/console/OctoberManifest.php @@ -0,0 +1,170 @@ +option('minBuild') ?? 420; + $maxBuild = $this->option('maxBuild') ?? 9999; + + $targetFile = (substr($this->argument('target'), 0, 1) === '/') + ? $this->argument('target') + : getcwd() . '/' . $this->argument('target'); + + if (empty($targetFile)) { + throw new ApplicationException( + 'A target argument must be specified for the generated manifest file.' + ); + } + + if ($minBuild > $maxBuild) { + throw new ApplicationException( + 'Minimum build specified is larger than the maximum build specified.' + ); + } + + if (file_exists($targetFile)) { + $manifest = new SourceManifest($targetFile); + $manifestMaxBuild = $manifest->getMaxBuild(); + + if ($manifestMaxBuild > $minBuild) { + $minBuild = $manifestMaxBuild + 1; + + if ($minBuild > $maxBuild) { + throw new ApplicationException( + 'This manifest already contains all requested builds.' + ); + } + } + } else { + $manifest = new SourceManifest('', false); + } + + // Create temporary directory to hold builds + $buildDir = storage_path('temp/builds/'); + if (!is_dir($buildDir)) { + mkdir($buildDir, 0775, true); + } + + for ($build = $minBuild; $build <= $maxBuild; ++$build) { + // Download version from GitHub + $this->comment('Processing build ' . $build); + $this->line(' - Downloading...'); + + if (file_exists($buildDir . 'build-' . $build . '.zip') || is_dir($buildDir . $build . '/')) { + $this->info(' - Already downloaded.'); + } else { + $zipUrl = sprintf($this->sourceBuildFile, $build); + $zipFile = @file_get_contents($zipUrl); + + if (empty($zipFile)) { + $this->error(' - Not found.'); + break; + } + + file_put_contents($buildDir . 'build-' . $build . '.zip', $zipFile); + + $this->info(' - Downloaded.'); + } + + // Extract version + $this->line(' - Extracting...'); + if (is_dir($buildDir . $build . '/')) { + $this->info(' - Already extracted.'); + } else { + $zip = new ZipArchive; + if ($zip->open($buildDir . 'build-' . $build . '.zip')) { + $toExtract = []; + $paths = [ + 'october-1.0.' . $build . '/modules/backend/', + 'october-1.0.' . $build . '/modules/cms/', + 'october-1.0.' . $build . '/modules/system/', + ]; + + // Only get necessary files from the modules directory + for ($i = 0; $i < $zip->numFiles; ++$i) { + $filename = $zip->statIndex($i)['name']; + + foreach ($paths as $path) { + if (strpos($filename, $path) === 0) { + $toExtract[] = $filename; + break; + } + } + } + + if (!count($toExtract)) { + $this->error(' - Unable to get valid files for extraction. Cancelled.'); + exit(1); + } + + $zip->extractTo($buildDir . $build . '/', $toExtract); + $zip->close(); + + // Remove ZIP file + unlink($buildDir . 'build-' . $build . '.zip'); + } else { + $this->error(' - Unable to extract zip file. Cancelled.'); + exit(1); + } + + $this->info(' - Extracted.'); + } + + // Determine previous build + $manifestBuilds = $manifest->getBuilds(); + $previous = null; + if (count($manifestBuilds)) { + if (isset($manifestBuilds[$build - 1])) { + $previous = $build - 1; + } + } + + // Add build to manifest + $this->line(' - Adding to manifest...'); + $buildManifest = new FileManifest($buildDir . $build . '/october-1.0.' . $build); + $manifest->addBuild($build, $buildManifest, $previous); + $this->info(' - Added.'); + } + + // Generate manifest + $this->comment('Generating manifest...'); + file_put_contents($targetFile, $manifest->generate()); + + $this->comment('Completed.'); + } +} diff --git a/modules/system/console/OctoberUtil.php b/modules/system/console/OctoberUtil.php index 9a886d665..14700a31f 100644 --- a/modules/system/console/OctoberUtil.php +++ b/modules/system/console/OctoberUtil.php @@ -1,6 +1,5 @@ : Set the projectId for this october instance. * * @package october\system @@ -111,31 +108,10 @@ class OctoberUtil extends Command protected function utilSetBuild() { - $this->comment('-'); + $this->comment('NOTE: This command is now deprecated. Please use "php artisan october:version" instead.'); + $this->comment(''); - /* - * Skip setting the build number if no database is detected to set it within - */ - if (!App::hasDatabase()) { - $this->comment('No database detected - skipping setting the build number.'); - return; - } - - try { - $build = UpdateManager::instance()->setBuildNumberManually(); - $this->comment('*** October sets build: '.$build); - } - catch (Exception $ex) { - $this->comment('*** You were kicked from #october by Ex: ('.$ex->getMessage().')'); - } - - $this->comment('-'); - sleep(1); - $this->comment('Ping? Pong!'); - $this->comment('-'); - sleep(1); - $this->comment('Ping? Pong!'); - $this->comment('-'); + return $this->call('october:version'); } protected function utilCompileJs() diff --git a/modules/system/console/OctoberVersion.php b/modules/system/console/OctoberVersion.php new file mode 100644 index 000000000..95a809b58 --- /dev/null +++ b/modules/system/console/OctoberVersion.php @@ -0,0 +1,47 @@ +comment('*** Detecting October CMS build...'); + + if (!App::hasDatabase()) { + $build = UpdateManager::instance()->getBuildNumberManually(); + + // Skip setting the build number if no database is detected to set it within + $this->comment('*** No database detected - skipping setting the build number.'); + } else { + $build = UpdateManager::instance()->setBuildNumberManually(); + } + + if (is_null($build)) { + $this->error('Unable to detect your build of October CMS.'); + return; + } + + if ($build['modified']) { + $this->info('*** Detected a modified version of October CMS build ' . $build['build'] . '.'); + } else { + $this->info('*** Detected October CMS build ' . $build['build'] . '.'); + } + } +} diff --git a/modules/system/controllers/Updates.php b/modules/system/controllers/Updates.php index ed66175d5..1d1be7923 100644 --- a/modules/system/controllers/Updates.php +++ b/modules/system/controllers/Updates.php @@ -75,6 +75,7 @@ class Updates extends Controller public function index() { $this->vars['coreBuild'] = Parameter::get('system::core.build'); + $this->vars['coreBuildModified'] = Parameter::get('system::core.modified', false); $this->vars['projectId'] = Parameter::get('system::project.id'); $this->vars['projectName'] = Parameter::get('system::project.name'); $this->vars['projectOwner'] = Parameter::get('system::project.owner'); diff --git a/modules/system/controllers/updates/index.htm b/modules/system/controllers/updates/index.htm index f59abe07c..64f2b831a 100644 --- a/modules/system/controllers/updates/index.htm +++ b/modules/system/controllers/updates/index.htm @@ -34,7 +34,19 @@