From bde616fc60de3d053a58dbf45242cdffa44035d0 Mon Sep 17 00:00:00 2001 From: Markus Birth Date: Thu, 15 Feb 2018 00:51:51 +0100 Subject: [PATCH] Rewrite to Python. --- ansi.py | 29 +++++ apt-urlcheck.php | 279 ----------------------------------------------- apt-urlcheck.py | 116 ++++++++++++++++++++ requirements.txt | 3 + 4 files changed, 148 insertions(+), 279 deletions(-) create mode 100644 ansi.py delete mode 100755 apt-urlcheck.php create mode 100755 apt-urlcheck.py create mode 100644 requirements.txt diff --git a/ansi.py b/ansi.py new file mode 100644 index 0000000..a6ad41d --- /dev/null +++ b/ansi.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +"""Listing of ANSI colors plus additional Windows support.""" + +import platform + +# Needed to make ANSI escape sequences work in Windows +SYSTEM = platform.system() +if SYSTEM == "Windows": + try: + import colorama + colorama.init() + except ImportError: + pass + +UP_DEL = u"\u001b[F\u001b[K" +BLACK = u"\u001b[0;30m" +RED_DARK = u"\u001b[0;31m" +GREEN_DARK = u"\u001b[0;32m" +YELLOW_DARK = u"\u001b[0;33m" +CYAN_DARK = u"\u001b[0;36m" +SILVER = u"\u001b[0;37m" +GREY = u"\u001b[1;30m" +RED = u"\u001b[1;31m" +GREEN = u"\u001b[1;32m" +YELLOW = u"\u001b[1;33m" +CYAN = u"\u001b[1;36m" +WHITE = u"\u001b[1;37m" +RESET = u"\u001b[0m" diff --git a/apt-urlcheck.php b/apt-urlcheck.php deleted file mode 100755 index 9a42819..0000000 --- a/apt-urlcheck.php +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/php -codename = exec('lsb_release -cs'); - - $this->addSourceLists( '/etc/apt/sources.list' ); - $this->addSourceLists( '/etc/apt/sources.list.d/*.list' ); - } - - public function addSourceLists( $filemask ) - { - $this->aptlists = array_merge( $this->aptlists, glob( $filemask ) ); - } - - public function getSourceLists() - { - return $this->aptlists; - } - - public function addWhitelist( $codename ) - { - if ( !is_array( $codename ) ) $codename = array( $codename ); - foreach ( $codename as $cn ) { - $this->whitelist[] = $cn; - } - } - - public function isRoot() - { - return (posix_getuid() == 0); - } - - public function getCodename() - { - return $this->codename; - } - - public function parseLists() - { - $debs = array(); - foreach ( $this->getSourceLists() as $path ) { - $fc = file($path, FILE_IGNORE_NEW_LINES|FILE_SKIP_EMPTY_LINES); - foreach ($fc as $i=>$fline) { - if ((substr($fline, 0, 4) == 'deb ') || (substr($fline, 0, 8) == 'deb-src ')) { - $debs[] = array( - 'file' => $path, - 'line' => $i+1, - 'data' => $fline, - ); - } - } - } - return $debs; - } - - public function parseDebLine( $line ) - { - $parts = explode(' ', $line); - if ( $parts[1]{0} == '[' ) { - $result = array( - 'type' => $parts[0], - 'attributes' => $parts[1], - 'url' => $parts[2], - 'distr' => $parts[3], - 'components' => array_slice($parts, 4), - ); - } else { - $result = array( - 'type' => $parts[0], - 'attributes' => '', - 'url' => $parts[1], - 'distr' => $parts[2], - 'components' => array_slice($parts, 3), - ); - } - return $result; - } - - private function tryFetch( $url ) { - $answer = array(); - stream_context_set_default( array( - 'http' => array( - 'method' => 'HEAD', - 'timeout' => 5.0, - ), - ) ); - $result = get_headers( $url, 1 ); - $status_line = $result[0]; - if ( is_array( $status_line ) ) { - $status_line = end( $status_line ); - } - $status = substr( $status_line, 9, 3 ); - - stream_context_set_default( array( - 'http' => array( - 'method' => 'GET', - ), - ) ); - - return ( $status == '200' ); - } - - private function tryGetDirectoryListing( $url ) - { - $all_known_codenames = array_merge($this->codenames_old, $this->codenames); - $list = @file_get_contents( $url ); - if ( $list === false ) return false; - preg_match_all('//i', $list, $matches); - #print_r($matches); - $result = array(); - foreach ($matches[1] as $match) { - if ($match{0} != '?' && $match{0} != '/' && substr($match, -1) == '/' && $match != '../' && substr($match, 0, 4) != 'http') { - $result[] = substr($match, 0, -1); - } elseif (in_array($match, $all_known_codenames) || in_array(substr($match, 0, -1), $all_known_codenames)) { - $result[] = $match; - } - } - #print_r($result); - return array_unique( $result ); - } - - private function tryReleases( $baseurl, $additional = array() ) - { - if ( !is_array( $additional ) ) $additional = array( $additional ); - - $result = array(); - foreach ( array_unique( array_merge( $this->codenames, $additional ) ) as $codename ) { - foreach ( array( 'InRelease', 'Release', 'Release.gpg' ) as $filename ) { - $try_url = $baseurl . '/dists/' . $codename . '/' . $filename; - $exists = $this->tryFetch( $try_url ); - if ( $exists !== false ) { - $result[] = $codename; - break; - } - } - } - return $result; - } - - private function getServerInfo( $info ) - { - $result = array(); - $result['method'] = 'dirlist'; - $dirlist = $this->tryGetDirectoryListing( $info['url'] . '/dists' ); - if ( $dirlist === false ) { - // Damn! - $dirlist = $this->tryReleases( $info['url'], $info['distr'] ); - $result['method'] = 'bruteforce'; - } - $result['dists'] = $dirlist; - return $result; - } - - public function analyzeDebs( $debs = null, $progress = null ) - { - if (is_null($debs)) $debs = $this->parseLists(); - $result = array(); - foreach ( $debs as $deb ) { - $info = $this->parseDebLine( $deb['data'] ); - if ( substr( $info['distr'], 0, strlen( $this->codename ) ) != $this->codename && !in_array( $info['distr'], $this->whitelist ) ) { - $serverinfo = $this->getServerInfo( $info ); - $result[] = array( - 'deb' => $deb, - 'info' => $info, - 'server' => $serverinfo, - ); - } - if ($this->isRoot()) $this->getKeyId($info['url'], $info['distr']); // cache key-id - if (!is_null($progress)) call_user_func($progress); - } - return $result; - } - - private function getKeyId( $url, $dist ) - { - global $keycache; - if (!isset($keycache)) $keycache = array(); - if (isset($keycache[$url][$dist])) return $keycache[$url][$dist]; - $sigfile = @file_get_contents($url.'/dists/'.$dist.'/Release.gpg'); - if ($sigfile === false) { - echo 'No signature found.' . PHP_EOL; - $keycache[$url][$dist] = false; - return false; - } - exec('echo "' . $sigfile . '" | gpg --batch --verify - /etc/passwd 2>&1', $output, $retval); - $output = implode(PHP_EOL, $output); - preg_match('/gpg: Signature made (.*) using (.*) key ID (.*)\r?\n/m', $output, $matches); - $result = array( - 'date' => $matches[1], - 'type' => $matches[2], - 'id' => $matches[3], - 'desc' => $matches[4], - ); - echo 'Repo uses key id ' . $result['id'] . PHP_EOL; - if (empty($result['id'])) { - echo 'DEBUG: ' . $output . PHP_EOL; - } - $keycache[$url][$dist] = $result; - return $result; - } - - public function outputResults( $debinfo ) { - $all_codenames = array_merge( $this->codenames_old, $this->codenames ); - foreach ( $debinfo as $di ) { - echo 'Mismatching distribution "' . $di['info']['distr'] . '" in ' . $di['deb']['file'] . ':' . $di['deb']['line'] . PHP_EOL; - $better = array(); - $current = array_search( $di['info']['distr'], $all_codenames ); - foreach ( $di['server']['dists'] as $dist_avail ) { - $where = array_search( $dist_avail, $all_codenames ); - if ( $where === false || $where > $current ) { - $better[] = $dist_avail; - } - } - //echo 'Available: ' . implode( ', ', $di['server']['dists'] ) . PHP_EOL; - if ( count( $better ) > 0 ) echo 'Possibly better matches: ' . implode( ', ', $better ) . PHP_EOL; - } - } -} - -$ac = new APTChecker(); -$ac->addWhitelist( array( 'stable', 'unstable', 'beta' ) ); - -echo 'System: ' . $ac->getCodename() . PHP_EOL; - -echo 'Running as user id ' . posix_getuid() . PHP_EOL; - -echo 'Parsing lists ... '; -$debs = $ac->parseLists(); -echo count($debs) . ' entries found.' . PHP_EOL; - -if ( $ac->isRoot() ) { - echo 'Fetching key ids of keyring ... '; - exec('apt-key list', $output, $retval); - $output = implode(PHP_EOL, $output); - preg_match_all('/pub.*\/([^\s]+).*\r?\n/m', $output, $matches); - $allkeys = $matches[1]; - echo count($allkeys) . ' keys found.' . PHP_EOL; -} else { - echo 'Not started as root. Will not check GPG keys.' . PHP_EOL; -} - -$debinfo = $ac->analyzeDebs( $debs, 'progressInd' ); -echo PHP_EOL; -$ac->outputResults( $debinfo ); - -if ( $ac->isRoot() ) { - foreach ( $keycache as $url=>$dists ) { - foreach ( $dists as $dist=>$key ) { - if ( !empty( $key['id'] ) && !in_array( $key['id'], $allkeys ) ) { - echo 'Importing key ' . $key['id'] . ' ... '; - passthru( 'apt-key adv --batch --recv-keys --keyserver keyserver.ubuntu.com ' . $key['id'] ); - } - } - } -} - -echo 'All done.'; -if ( !$ac->isRoot() ) echo ' (Run as root to import missing PPA keys.)'; -echo PHP_EOL; -exit; - -function progressInd() { - echo '.'; -} - -function checkCurrentDist($url) { - $fullurl = 'http://linux.getdropbox.com/ubuntu/dists/jaunty/main/binary-i386/Packages.gz'; -} - diff --git a/apt-urlcheck.py b/apt-urlcheck.py new file mode 100755 index 0000000..81f5065 --- /dev/null +++ b/apt-urlcheck.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import ansi +import os.path +import re +import requests +from aptsources.distro import get_distro +from aptsources.sourceslist import SourcesList +from typing import TypeVar + +codenames_okay = ["devel", "stable", "unstable", "beta", "preview", "testing", "syncthing"] +codenames_old = ["gutsy", "hardy", "intrepid", "jaunty", "karmic", "lucid", "maverick", "natty", "oneiric", "precise", "quantal", "raring", "saucy", "trusty", "utopic", "vivid", "wily"] +codenames = ["jessie", "xenial", "yakkety", "zesty", "stretch", "artful", "bionic", "squeeze"] + +codename = get_distro().codename +print("This is {}.".format(ansi.YELLOW + codename + ansi.RESET)) + +print("Loading sources...", end="", flush=True) + +valid_sources = 0 +outdated_sources = 0 +check_sources = [] +for source in SourcesList(): + if source.disabled or source.line=="\n": + continue + valid_sources += 1 + #print(".", end="", flush=True) + if codename in source.dist or source.dist in codenames_okay: + continue + outdated_sources += 1 + check_sources.append(source) + #print("{}: {}".format(source.file, source.line.strip())) + +check_sources = sorted(check_sources, key=lambda x: x.file) + +print(" OK") +print("Found {} sources with {} possibly outdated.".format(valid_sources, outdated_sources)) + + +BorL = TypeVar("BorL", bool, list) + +fetch_cache = {} + +def try_fetch_dirlisting(url: str) -> BorL: + global codenames_okay, codenames_old, codenames, fetch_cache + if url in fetch_cache: + return fetch_cache[url] + all_known_codenames = codenames_old + codenames + result = requests.get(url) + if result.status_code != 200: + fetch_cache[url] = False + return False + matches = re.findall(r"", result.text) + valid_matches = [] + for match in matches: + if match[0] != "?" and match[0] != "/" and match[-1:] == "/" and match != "../" and match[0:4] != "http": + valid_matches.append(match[0:-1]) + elif match in all_known_codenames or match[0:-1] in all_known_codenames: + valid_matches.append(match) + fetch_cache[url] = valid_matches + return fetch_cache[url] + +def mutate_codename(current_codename: str, new_codename: str) -> str: + global codenames_okay, codenames_old, codenames + all_codenames = codenames_old + codenames + for cn in all_codenames: + if cn in current_codename: + return current_codename.replace(cn, new_codename) + return new_codename + +probe_cache = {} + +def try_url_probing(url: str, current_codename: str) -> list: + global codenames_okay, codenames_old, codenames, probe_cache + cache_key = url + "|" + current_codename + if cache_key in probe_cache: + return probe_cache[cache_key] + test_set = codenames + codenames_okay + valid_matches = [] + for codename in test_set: + for filename in ["InRelease", "Release", "Release.gpg"]: + mcodename = mutate_codename(current_codename, codename) + try_url = "{}/{}/{}".format(url, mcodename, filename) + print(".", end="", flush=True) + result = requests.get(try_url) + if result.status_code == 200: + valid_matches.append(mcodename) + break + probe_cache[cache_key] = valid_matches + return probe_cache[cache_key] + +def filter_better_matches(found_codenames: list, current_codename: str) -> list: + global codenames, codenames_okay, codenames_old + better_codenames = [] + for cn in found_codenames: + if cn > current_codename: + better_codenames.append(cn) + elif cn in codenames_okay: + better_codenames.append(cn) + return better_codenames + +for src in check_sources: + print("{}: Outdated codename: {}".format(ansi.CYAN + os.path.basename(src.file) + ansi.RESET, ansi.RED + src.dist + ansi.RESET)) + test_url = src.uri + "/dists" + more_options = try_fetch_dirlisting(test_url) + if not more_options: + print("Listing failed. Probing", end="", flush=True) + more_options = try_url_probing(test_url, src.dist) + print(" OK") + print(ansi.UP_DEL, end="") + better_options = filter_better_matches(more_options, src.dist) + if better_options: + print("Possibly better options: {}".format(ansi.GREEN + (ansi.RESET + ", " + ansi.GREEN).join(better_options) + ansi.RESET)) + else: + print(ansi.SILVER + "No better match(es) found at the moment." + ansi.RESET) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4b44bb1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# Install via aptitude, pip doesn't work! +python-apt +python-distutils-extra