Merge pull request #489 from InQuest/rc

InQuest · Sep 20, 2023 · 79f76f1 · 79f76f1
2 parents 742a778 + 82e1745
commit 79f76f1
Show file tree

Hide file tree

Showing 33 changed files with 254 additions and 320 deletions.
diff --git a/wiki/res/inquest_logo.svg → .github/wiki/inquest_logo.svg b/wiki/res/inquest_logo.svg → .github/wiki/inquest_logo.svg
diff --git a/wiki/res/threatkb-db-struct.svg → .github/wiki/threatkb-db-struct.svg b/wiki/res/threatkb-db-struct.svg → .github/wiki/threatkb-db-struct.svg
diff --git a/.gitignore b/.gitignore
@@ -70,3 +70,5 @@ dump.rdb
 # STS (Spring Tool Suite)
 .springBeans
 .vagrant
+
+version
diff --git a/README.md b/README.md
@@ -1,41 +1,35 @@
-
 <p align="center">
-    <img src="wiki/res/inquest_logo.svg"></img>
+  <img src="https://raw.githubusercontent.com/InQuest/ThreatKB/rc/.github/wiki/inquest_logo.svg" />
 </p>
 
-
 ### NOTE: THIS REPO IS IN AN ALPHA STATE
 
-ThreatKB is a knowledge base workflow management dashboard for Yara rules and C2 artifacts. Rules are categorized and used to denote intent, severity, and confidence on accumulated artifacts.
+ThreatKB is a knowledge base workflow management dashboard for YARA rules and C2 artifacts. Rules are categorized and used to denote intent, severity, and confidence in accumulated artifacts.
 
-To start using ThreatKB, follow our [guide](wiki/setup.md).
+To start using ThreatKB, check out our [wiki](https://github.com/InQuest/ThreatKB/wiki).
 
-  ---  
+---
 
 Installing by Docker is the currently recommended way of setting up ThreatKB, directions are included as the first link in the wiki. Installation by source is included in the wiki as well.
 
-
 ## Table of Contents
 
-* [Docker Installation](wiki/docker.md)
-* [Setup ThreatKB](wiki/setup.md)
-  + [Pre-requisites](wiki/setup.md#pre-requisites)
-  + [System Prep](wiki/setup.md#system-prep)
-* [Getting Started](wiki/getting-started.md)
-  + [Application Install](wiki/getting-started.md#application-install)
-  + [Running ThreatKB](wiki/getting-started.md#running-threatkb)
-  + [Admin User Creation](wiki/getting-started.md#admin-user-creation)
-* [Databases](wiki/db-struct.md)
-* [Documentation](wiki/documentation.md)
-* [FAQ](wiki/faq.md)
-
-
+* [Home](https://github.com/InQuest/ThreatKB/wiki)
+* [Setup ThreatKB](https://github.com/InQuest/ThreatKB/wiki/Setup)
+  + [Pre-requisites](https://github.com/InQuest/ThreatKB/wiki/Setup#pre-requisites)
+  + [System Prep](https://github.com/InQuest/ThreatKB/wiki/Setup#system-prep)
+  + [Application Install](https://github.com/InQuest/ThreatKB/wiki/Setup#application-install)
+* [Getting Started](https://github.com/InQuest/ThreatKB/wiki/Getting-Started)
+  + [Running ThreatKB](https://github.com/InQuest/ThreatKB/wiki/Getting-Started#running-threatkb)
+  + [Admin User Creation](https://github.com/InQuest/ThreatKB/wiki/Getting-Started#admin-user-creation)
+* [Docker Installation](https://github.com/InQuest/ThreatKB/wiki/Docker)
+* [Database Structure](https://github.com/InQuest/ThreatKB/wiki/Database-Structure)
+* [Documentation](https://github.com/InQuest/ThreatKB/wiki/Documentation)
+* [FAQ](https://github.com/InQuest/ThreatKB/wiki/Frequently-Asked-Questions)
 
 ## Thank You
-ThreatKB utilizes Plyara to parse yara rules into python dictionaries. A huge thank you to the Plyara team! Links to the project are below:
+ThreatKB utilizes Plyara to parse YARA rules into Python dictionaries. A huge thank you to the Plyara team! Links to the project are below:
 
-https://github.com/8u1a/plyara
-https://github.com/8u1a/plyara/blob/master/LICENSE
+- [Plyara](https://github.com/plyara/plyara) ([LICENSE](https://github.com/plyara/plyara/blob/master/LICENSE))
 
 When a release is created, the system first pulls all signatures that are in the release state. Then, it gathers all signatures that are in the staging state and checks their revision history for the most recently released revision that is in the release state. If it finds it, it will include it in the release. If it does not find any previously released revisions, it will skip the signature.
-
diff --git a/app/__init__.py b/app/__init__.py
@@ -1,3 +1,8 @@
+import os
+import sys
+import logging
+import datetime
+import distutils
 import functools
 
 from flask import Flask, abort, jsonify, request
@@ -10,14 +15,19 @@
 from functools import wraps, update_wrapper
 from flask_caching import Cache
 from flask_selfdoc import Autodoc
-import datetime
-import logging
-import os
-import distutils
 
 
 app = Flask(__name__, static_url_path='')
-app.config.from_object("config")
+
+if os.path.exists("testing_config.py") and not os.path.exists("config.py"):
+    app.config.from_object("testing_config")
+
+if os.path.exists("config.py"):
+    app.config.from_object("config")
+
+if not os.path.exists("config.py") and not os.path.exists("testing_config.py"):
+    print("Missing configuration settings")
+    sys.exit(1)
 
 if app.config.get('REDIS_CACHE_URL', None):
     cache = Cache(app, config={'CACHE_TYPE': 'redis', 'CACHE_REDIS_URL': app.config.get('REDIS_CACHE_URL')})
@@ -203,7 +213,6 @@ def teardown_request(exception):
     db.session.remove()
 
 
-@app.before_first_request
 def setup_logging():
     app.logger.addHandler(logging.StreamHandler())
     app.logger.setLevel(app.config["LOGGING_LEVEL"])
@@ -304,7 +313,6 @@ def teardown_request(exception):
             db.session.rollback()
         db.session.remove()
 
-    @app.before_first_request
     def setup_logging():
         app.logger.addHandler(logging.StreamHandler())
         app.logger.setLevel(app.config["LOGGING_LEVEL"])
@@ -333,6 +341,9 @@ def load_user_from_request(request):
 
         return None
 
+    with app.app_context():
+        setup_logging()
+
 
 def run(debug=False, port=0, host=''):
     import os
@@ -342,3 +353,7 @@ def run(debug=False, port=0, host=''):
 
     generate_app()
     app.run(debug=debug, port=port, host=host)
+
+
+with app.app_context():
+    setup_logging()
diff --git a/app/models/users.py b/app/models/users.py
@@ -1,5 +1,4 @@
-from itsdangerous import (JSONWebSignatureSerializer
-                          as Serializer, BadSignature)
+from itsdangerous import (URLSafeTimedSerializer as Serializer, BadSignature)
 
 from app import db
 from app.models import metadata, scripts

diff --git a/app/routes/access_keys.py b/app/routes/access_keys.py
@@ -1,3 +1,4 @@
+import codecs
 import time
 import datetime
 import os
@@ -76,7 +77,7 @@ def create_access_key():
     active_inactive_count = json.loads(get_active_inactive_key_count().data)
 
     if active_inactive_count and active_inactive_count['activeInactiveCount'] < 2:
-        s_key = os.urandom(24).encode('hex')
+        s_key = codecs.encode(os.urandom(24), 'hex').decode()
         token = user.generate_auth_token(s_key)
 
         key = AccessKeys(

diff --git a/app/routes/batch.py b/app/routes/batch.py
@@ -37,6 +37,8 @@ def batch_update(batch, artifact, session, include_tags=True):
     fields_to_update = dict()
     if 'description' in batch and batch['description']:
         fields_to_update['description'] = batch['description']
+    if 'match_type' in batch and batch['match_type']:
+        fields_to_update['match_type'] = batch['match_type']
     if 'expiration_timestamp' in batch and batch['expiration_timestamp'] and hasattr(artifact, 'expiration_timestamp'):
         fields_to_update['expiration_timestamp'] = batch['expiration_timestamp']
     if 'category' in batch and batch['category'] and hasattr(artifact, 'category'):

diff --git a/app/routes/c2dns.py b/app/routes/c2dns.py
@@ -234,6 +234,7 @@ def batch_update_c2dns():
     """Batch update c2dns artifacts
     From Data: batch {
                  state (str),
+                 match_type (str),
                  owner_user (str),
                  tags (array),
                  ids (array)

diff --git a/app/routes/files.py b/app/routes/files.py
@@ -254,9 +254,7 @@ def get_file_for_entity(entity_type, entity_id, file_id):
     if not file_entity:
         abort(404)
 
-    full_path = os.path.join(cfg_settings.Cfg_settings.get_setting("FILE_STORE_PATH"),
-                             str(ENTITY_MAPPING[entity_type]) if entity_type != "0" else "",
-                             str(entity_id) if entity_id != 0 else "",
+    full_path = os.path.join(file_entity.full_path,
                              secure_filename(file_entity.filename))
     if not os.path.exists(full_path):
         abort(404)

diff --git a/app/routes/import_.py b/app/routes/import_.py
@@ -13,7 +13,7 @@
 #####################################################################
 
 def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared_reference=None,
-                   shared_description=None, shared_state=None, shared_owner=None, metadata_field_mapping={},
+                   shared_description=None, shared_state=None, shared_match_type=None, shared_owner=None, metadata_field_mapping={},
                    resurrect_retired_artifacts=True):
     default_state = "Imported"
     return_artifacts = []
@@ -134,6 +134,7 @@ def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared
                                               entity_id=dns.id, user_id=current_user.id))
                     dns.created_user_id, dns.modified_user_id = current_user.id, current_user.id
                     dns.state = default_state if not shared_state else shared_state
+                    dns.match_type = None if not shared_match_type else shared_match_type
                     if Whitelist.hits_whitelist(dns.domain_name, dns.state):
                         error_artifacts.append((dns.domain_name, f"Whitelist validation failed {dns.domain_name}"))
                         continue
@@ -144,6 +145,8 @@ def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared
                         dns.description = shared_description
                     if shared_state:
                         dns.state = shared_state
+                    if shared_match_type:
+                        dns.match_type = shared_match_type
                     if shared_owner:
                         dns.owner_user_id = shared_owner
 
@@ -239,6 +242,7 @@ def import_artifacts():
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     import_text = request.json.get('import_text', None)
     shared_state = request.json.get('shared_state', None)
+    shared_match_type = request.json.get('shared_match_type', None)
     shared_reference = request.json.get("shared_reference", None)
     shared_description = request.json.get("shared_description", None)
     shared_owner = request.json.get("shared_owner", None)
@@ -259,7 +263,7 @@ def import_artifacts():
     if autocommit:
         artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                    artifacts=artifacts, shared_reference=shared_reference,
-                                   shared_description=shared_description, shared_state=shared_state,
+                                   shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                    shared_owner=shared_owner, metadata_field_mapping=metadata_field_mapping,
                                    resurrect_retired_artifacts=resurrect_retired_artifacts)
 
@@ -280,6 +284,7 @@ def import_artifacts_by_filek():
     import_text = request.files['file'].stream.read()
     import_text = import_text.strip()
     shared_state = request.values.get('shared_state', None)
+    shared_match_type = request.values.get('shared_match_type', None)
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     shared_reference = request.values.get("shared_reference", None) or None
     shared_description = request.values.get("shared_description", None) or None
@@ -301,7 +306,7 @@ def import_artifacts_by_filek():
     if autocommit:
         artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                    artifacts=artifacts, shared_reference=shared_reference,
-                                   shared_description=shared_description, shared_state=shared_state,
+                                   shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                    shared_owner=shared_owner, metadata_field_mapping=metadata_field_mapping,
                                    resurrect_retired_artifacts=resurrect_retired_artifacts)
 
@@ -322,6 +327,7 @@ def commit_artifacts():
     shared_description = request.json.get("shared_description", None)
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     shared_state = request.json.get('shared_state', None)
+    shared_match_type = request.json.get('shared_match_type', None)
     extract_ip = request.json.get('extract_ip', True)
     extract_dns = request.json.get('extract_dns', True)
     shared_owner = request.json.get("shared_owner", None)
@@ -336,7 +342,7 @@ def commit_artifacts():
 
     artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                artifacts=artifacts, shared_reference=shared_reference,
-                               shared_description=shared_description, shared_state=shared_state,
+                               shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                metadata_field_mapping=metadata_field_mapping, shared_owner=shared_owner,
                                resurrect_retired_artifacts=resurrect_retired_artifacts)
     return jsonify({"artifacts": artifacts}), 201
diff --git a/app/routes/version.py b/app/routes/version.py
@@ -11,7 +11,10 @@ def get_version():
     Return: version number"""
     try:
         version = [v.strip() for v in open("version", "r").readlines()]
+        return Response(
+            json.dumps({"version": version[0], "version_email": version[1], "version_date": version[2][:-6]}),
+            mimetype='application/json')
     except:
-        version = ["unavailable", "unavailable", "unavailable"]
-    return Response(json.dumps({"version": version[0], "version_email": version[1], "version_date": version[2][:-6]}),
-                    mimetype='application/json')
+        return Response(
+            json.dumps({"version": "unavailable", "version_email": "unavailable", "version_date": "unavailable"}),
+            mimetype='application/json')
diff --git a/app/routes/yara_rules.py b/app/routes/yara_rules.py
@@ -599,7 +599,7 @@ def update_yara_rule(id):
     if tags_to_create:
         create_tags_mapping(entity.__tablename__, entity.id, tags_to_create)
 
-    return jsonify(entity.to_dict()), 200
+    return jsonify(entity.to_dict(include_yara_rule_string=True)), 200
 
 
 @app.route('/ThreatKB/yara_rules/batch/edit', methods=['PUT'])

diff --git a/app/static/js/c2dns/c2dns-controller.js b/app/static/js/c2dns/c2dns-controller.js
@@ -420,6 +420,7 @@ angular.module('ThreatKB')
                 var c2dnsToUpdate = {
                     owner_user: $scope.batch.owner,
                     state: $scope.batch.state,
+                    match_type: $scope.batch.match_type,
                     description: $scope.batch.description,
                     expiration_timestamp: $scope.batch.expiration_timestamp,
                     tags: $scope.batch.tags,
@@ -505,6 +506,7 @@ angular.module('ThreatKB')
                 $scope.batch = {
                     owner: null,
                     state: null,
+                    match_type: null,
                     description: null,
                     expiration_timestamp: null,
                     tags: null
@@ -829,6 +831,8 @@ angular.module('ThreatKB')
 
             $scope.cfg_states = Cfg_states.query();
 
+            $scope.match_types = ['exact', 'wildcard'];
+
             $scope.ok = function () {
                 $uibModalInstance.close($scope.batch);
             };

diff --git a/app/static/js/import/import-controller.js b/app/static/js/import/import-controller.js
@@ -6,6 +6,8 @@ angular.module('ThreatKB').controller('ImportController',
 
             $scope.cfg_states = Cfg_states.query();
             $scope.shared_state = {};
+            $scope.shared_match_type = {};
+            $scope.match_types = ['exact', 'wildcard'];
             $scope.shared_owner = null;
             $scope.users = Users.query();
             $scope.default_mapping = Cfg_settings.get({key: "DEFAULT_METADATA_MAPPING"});
@@ -39,6 +41,7 @@ angular.module('ThreatKB').controller('ImportController',
                                     shared_reference: $scope.shared_reference,
                                     shared_description: $scope.shared_description,
                                     shared_state: $scope.shared_state,
+                                    shared_match_type: $scope.shared_match_type,
                                     shared_owner: $scope.shared_owner,
                                     extract_ip: $scope.extract_ip,
                                     extract_dns: $scope.extract_dns,
@@ -89,6 +92,9 @@ angular.module('ThreatKB').controller('ImportController',
                     $scope.shared_state.state = {};
                 }
 
+                if ($scope.shared_match_type === undefined) {
+                    $scope.shared_match_type = {};
+                }
                 var artifacts_to_commit = [];
                 for (var i = 0; i < $scope.artifacts.length; i++) {
                     if ($scope.checked_indexes[i]) {
@@ -99,7 +105,7 @@ angular.module('ThreatKB').controller('ImportController',
                 blockUI.start($scope.block_message);
 
                 var field_mapping = JSON.parse($scope.default_mapping.value);
-                Import.commit_artifacts(artifacts_to_commit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
+                Import.commit_artifacts(artifacts_to_commit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_match_type.match_type, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
                     blockUI.stop();
                     var ttl = 3000;
                     var message = "";
@@ -137,12 +143,15 @@ angular.module('ThreatKB').controller('ImportController',
                     $scope.shared_state.state = {};
                 }
 
+                if ($scope.shared_match_type === undefined) {
+                    $scope.shared_match_type = {};
+                }
                 if ($scope.autocommit) {
                     blockUI.start($scope.block_message);
                 }
 
                 var field_mapping = JSON.parse($scope.default_mapping.value);
-                Import.import_artifacts($scope.import_text, $scope.autocommit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
+                Import.import_artifacts($scope.import_text, $scope.autocommit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_match_type.match_type, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
                     if ($scope.autocommit) {
                         blockUI.stop();
                         var message = "";
@@ -206,6 +215,7 @@ angular.module('ThreatKB').controller('ImportController',
                 $scope.users = Users.query();
                 $scope.cfg_states = Cfg_states.query();
                 $scope.shared_state = {};
+                $scope.shared_match_type = {};
                 $scope.extract_ip = true;
                 $scope.extract_dns = true;
                 $scope.extract_signature = true;