Initial experimental CVE checker (port of TingPing's script)

src/checker.py

@@ -1,7 +1,9 @@
 # Copyright (C) 2018 Endless Mobile, Inc.
 #
 # Authors:
+#       Andrew Hayzen <ahayzen@gmail.com>
 #       Joaquim Rocha <jrocha@endlessm.com>
+#       Patrick Griffis <tingping@tingping.se>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,7 +20,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from collections import OrderedDict
-from lib.externaldata import CheckerRegistry, ExternalData
+from lib import CheckerRegistry, CVEData, ExternalData
 
 import json
 import os
@@ -29,9 +31,10 @@ class NoManifestCheckersFound(Exception):
 
 class ManifestChecker:
 
-    def __init__(self, manifest):
+    def __init__(self, manifest, experimental_cve_checker=False):
         self._manifest = manifest
         self._external_data = []
+        self._cve_data = []
 
         # Load and initialize checkers
         CheckerRegistry.load(os.path.join(os.path.dirname(__file__), 'checkers'))
@@ -42,68 +45,11 @@ def __init__(self, manifest):
             clean_manifest = re.sub(r'(^|\s)/\*.*?\*/', '', manifest_file.read())
             self._json_data = json.loads(clean_manifest, object_pairs_hook=OrderedDict)
 
-        self._collect_external_data()
+        # FIXME: should we always loads these or from CLI args ?
+        self._external_data = ExternalData.collect_external_data(self._json_data)
 
-    def _collect_external_data(self):
-        self._external_data = self._get_module_data_from_json(self._json_data) + \
-                              self._get_finish_args_extra_data_from_json(self._json_data)
-
-    def _get_finish_args_extra_data_from_json(self, json_data):
-        extra_data_prefix = '--extra-data='
-        external_data = []
-        extra_data_str = [arg for arg in json_data.get('finish-args', []) \
-                          if arg.startswith(extra_data_prefix)]
-
-        for extra_data in extra_data_str:
-            # discard '--extra-data=' prefix from the string
-            extra_data = extra_data[len(extra_data_prefix) + 1:]
-            info, url = extra_data.split('::')
-            name, sha256sum, size = info.split(':')
-            data_type = ExternalData.Type.EXTRA_DATA
-            ext_data = ExternalData(data_type, name, url, sha256sum, size, [])
-            external_data.append(ext_data)
-
-        return external_data
-
-    def _get_module_data_from_json(self, json_data):
-        external_data = []
-        for module in json_data.get('modules', []):
-            for source in module.get('sources', []):
-                url = source.get('url', None)
-                if not url:
-                    continue
-
-                name = source.get('filename')
-                if not name:
-                    name = source.get('dest-filename')
-                if not name:
-                    name = os.path.basename(url)
-
-                data_type = source.get('type')
-                data_type = self._translate_data_type(data_type)
-                if data_type is None:
-                    continue
-
-                sha256sum = source.get('sha256', None)
-                arches = source.get('only-arches', [])
-                size = source.get('size', -1)
-                checker_data = source.get('x-checker-data')
-
-                ext_data = ExternalData(data_type, name, url, sha256sum, size,
-                                        arches, checker_data)
-                external_data.append(ext_data)
-
-        return external_data
-
-    def _translate_data_type(self, data_type):
-        types = { 'file': ExternalData.Type.FILE,
-                  'archive': ExternalData.Type.ARCHIVE,
-                  'extra-data': ExternalData.Type.EXTRA_DATA}
-        return types.get(data_type)
-
-    def print_external_data(self):
-        for data in self._external_data:
-            print(data)
+        if experimental_cve_checker:
+            self._cve_data = CVEData.collect_cve_data(self._json_data)
 
     def check(self, filter_type=None):
         '''Perform the check for all the external data in the manifest
@@ -150,3 +96,18 @@ def get_outdated_external_data(self):
                 external_data.append(data)
 
         return external_data
+
+    def print_cve_data(self):
+        # Print out an rst table
+        print('========================= ========')
+        print('Library                   Version')
+        print('========================= ========')
+
+        for library in self._cve_data:
+            print('{:25} {:8}'.format(library.name, library.version))
+
+        print('========================= ========')
+
+    def print_external_data(self):
+        for data in self._external_data:
+            print(data)

src/checkers/debianrepochecker.py

@@ -36,8 +36,7 @@
 import re
 import urllib
 
-from lib.externaldata import ExternalData, CheckerRegistry, Checker
-from lib import utils
+from lib import CheckerRegistry, Checker, ExternalData, utils
 
 DEB_PACKAGES_URL = '{root}/dists/{dist}/{comp}/binary-{arch}/Packages'
 DEB_PACKAGES_XZ_URL = '{root}/dists/{dist}/{comp}/binary-{arch}/Packages.xz'

src/checkers/rotatingurlchecker.py

@@ -31,8 +31,7 @@
 
 import logging
 
-from lib.externaldata import ExternalData, CheckerRegistry, Checker
-from lib import utils
+from lib import CheckerRegistry, Checker, ExternalData, utils
 
 class RotatingURLChecker(Checker):
 

src/checkers/urlchecker.py

@@ -21,8 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-from lib.externaldata import ExternalData, CheckerRegistry, Checker
-from lib import utils
+from lib import CheckerRegistry, Checker, ExternalData, utils
 
 class URLChecker(Checker):
 

src/flatpak-external-data-checker

@@ -6,6 +6,7 @@
 # Copyright (C) 2018 Endless Mobile, Inc.
 #
 # Authors:
+#       Andrew Hayzen <ahayzen@gmail.com>
 #       Joaquim Rocha <jrocha@endlessm.com>
 #
 # This program is free software; you can redistribute it and/or modify
@@ -62,12 +63,15 @@ if __name__ == '__main__':
                         'data as json', action="store_true")
     parser.add_argument('--filter-type', help='Only check external data of the given type',
                         choices=['all', 'extra-data', 'file', 'archive'], default='all')
+    parser.add_argument("--experimental-cve-checker",
+                        help="Experimental: CVE checker", action="store_true")
     args = parser.parse_args()
 
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
 
-    manifest_checker = checker.ManifestChecker(args.manifest)
+    manifest_checker = checker.ManifestChecker(args.manifest,
+                                               args.experimental_cve_checker)
 
     ext_data_types = {'all': None,
                       'archive': ExternalData.Type.ARCHIVE,
@@ -79,6 +83,9 @@ if __name__ == '__main__':
         sys.stderr.write('No manifest checkers were found\n')
         exit(2)
 
+    if args.experimental_cve_checker:
+        manifest_checker.print_cve_data()
+
     if print_outdated_external_data(manifest_checker, as_json=args.json):
         exit(1)
 

src/lib/__init__.py

@@ -0,0 +1,53 @@
+# Copyright (C) 2018 Endless Mobile, Inc.
+#
+# Authors:
+#       Joaquim Rocha <jrocha@endlessm.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import pkgutil
+
+
+class Checker:
+
+    def check(self, external_data):
+        raise NotImplementedError()
+
+
+class CheckerRegistry:
+
+    _checkers = []
+
+    @staticmethod
+    def load(checkers_folder):
+        for _unused, modname, _unused in pkgutil.walk_packages([checkers_folder]):
+            pkg_name = os.path.basename(checkers_folder)
+            __import__(pkg_name + '.' + modname)
+
+    @classmethod
+    def register_checker(class_, checker):
+        if not issubclass(checker, Checker):
+            raise TypeError('{} is not a of type {}'.format(checker, Checker))
+
+        class_._checkers.append(checker)
+
+    @classmethod
+    def get_checkers(class_):
+        return class_._checkers
+
+
+from lib.cvedata import CVEData
+from lib.externaldata import ExternalData

src/lib/cvedata.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2018 Endless Mobile, Inc.
+#
+# Authors:
+#       Andrew Hayzen <ahayzen@gmail.com>
+#       Joaquim Rocha <jrocha@endlessm.com>
+#       Patrick Griffis <tingping@tingping.se>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from collections import namedtuple
+from operator import itemgetter
+import re
+
+
+class CVEData:
+    @staticmethod
+    def collect_cve_data(json_data):
+        Library = namedtuple('Library', ('name', 'version'))
+        libraries = []
+
+        for module in json_data.get('modules', []):
+            if type(module) is str:
+                continue
+
+            version = CVEData.extract_version(module.get('sources', []))
+
+            if version:
+                libraries.append(Library(module.get("name", None), version))
+
+        libraries.sort(key=itemgetter(0))  # Sort by name
+
+        return libraries
+
+    @staticmethod
+    def extract_version(sources):
+        for source in sources:
+            if source.get('type', None) == 'archive':
+                url = source.get("url", None)
+
+                if not url:
+                    continue
+
+                filename = url.rpartition('/')[2]
+                match = re.search(r'(\d+\.\d+(?:\.\d+)?)', filename)
+
+                if match:
+                    return match.groups()[-1]
+                else:
+                    raise ValueError('Version not found in {}'.format(sources))