Merge remote-tracking branch 'origin/plover2'

.github/workflows/pytest.yml

@@ -40,7 +40,6 @@ jobs:
     - name: Set up config files
       run: |
         scp araxconfig@araxconfig.rtx.ai:config_secrets.json /mnt/config/config_secrets.json
-        echo "http://localhost:5008/api/rtxkg2/v1.4" > /mnt/config/kg2_url_override.txt
 
 
     - name: Download databases into CICD instance
@@ -71,25 +70,17 @@ jobs:
       run: sudo docker build --no-cache=true --rm -t rtx:test DockerBuild/ -f DockerBuild/Merged-Dockerfile
 
     - name: Run docker container
-      run: sudo docker run -d -it --name rtx_test -v /mnt/data/orangeboard/databases:/mnt/data/orangeboard/databases -v /mnt/config/kg2_url_override.txt:/mnt/data/orangeboard/production/RTX/code/kg2_url_override.txt -v /mnt/config/config_secrets.json:/mnt/data/orangeboard/production/RTX/code/config_secrets.json -v /mnt/config/config_secrets.json:/mnt/data/orangeboard/kg2/RTX/code/config_secrets.json rtx:test
-
-#     - name: Add config.domain
-#       run: |
-#         sudo docker exec rtx_test bash -c "sudo -u rt bash -c 'cd /mnt/data/orangeboard/production/RTX/code && echo \'RTXTeam/RTX Github actions test suite\' > config.domain'"
-#         sudo docker exec rtx_test bash -c "sudo -u rt bash -c 'cd /mnt/data/orangeboard/kg2/RTX/code && echo \'RTXTeam/RTX Github actions test suite\' > config.domain'"
+      run: sudo docker run -d -it --name rtx_test -v /mnt/data/orangeboard/databases:/mnt/data/orangeboard/databases -v /mnt/config/config_secrets.json:/mnt/data/orangeboard/production/RTX/code/config_secrets.json rtx:test
 
     - name: Create symlinks
-      run: |
-        sudo docker exec rtx_test bash -c "sudo -u rt bash -c 'cd /mnt/data/orangeboard/production/RTX && python3 code/ARAX/ARAXQuery/ARAX_database_manager.py'"
-        sudo docker exec rtx_test bash -c "sudo -u rt bash -c 'cd /mnt/data/orangeboard/kg2/RTX && python3 code/ARAX/ARAXQuery/ARAX_database_manager.py'"
+      run: sudo docker exec rtx_test bash -c "sudo -u rt bash -c 'cd /mnt/data/orangeboard/production/RTX && python3 code/ARAX/ARAXQuery/ARAX_database_manager.py'"
 
     - name: Build KP info cache
       run: sudo docker exec rtx_test bash -c "cd /mnt/data/orangeboard/production/RTX/code/ARAX/ARAXQuery/Expand && python3 kp_info_cacher.py"
 
     - name: Start services
       run: |
         sudo docker exec rtx_test service apache2 start
-        sudo docker exec rtx_test service RTX_OpenAPI_kg2 start
         sudo docker exec rtx_test service RTX_OpenAPI_production start
         sudo docker exec rtx_test service RTX_Complete start
     

code/ARAX/ARAXQuery/Path_Finder/converter/EdgeExtractorFromPloverDB.py

@@ -5,43 +5,48 @@ class EdgeExtractorFromPloverDB:
 
     def __init__(self, plover_url):
         self.plover_url = plover_url
-        self.edges = {}
-        self.nodes = {}
+        self.pairs_to_edge_ids = {}
+        self.knowledge_graph = {'edges': {}, 'nodes': {}}
 
     def get_extractor_url(self):
         return self.plover_url
 
-    def get_edges(self, node1_name, node1_id, node2_name, node2_id, edge_name, arax_response):
-        edge_key_1 = f"{node1_id}_{node2_id}"
-        edge_key_2 = f"{node2_id}_{node1_id}"
-        if edge_key_1 in self.edges:
-            return self.edges[edge_key_1]
-        if edge_key_2 in self.edges:
-            return self.edges[edge_key_2]
-        endpoint = "/query"
-        data = {
-            "edges": {
-                edge_name: {
-                    "subject": node1_name,
-                    "object": node2_name
-                }
-            },
-            "nodes": {
-                node1_name: {
-                    "ids": [node1_id, node2_id]
-                },
-                node2_name: {
-                    "ids": [node1_id, node2_id]
-                }
-            },
-            "include_metadata": True,
-            "respect_predicate_symmetry": True
-        }
+    def get_edges(self, pairs, arax_response):
+        cached_pairs = []
+        i = 0
+        while i < len(pairs):
+            edge_key_1 = f"{pairs[i][0]}--{pairs[i][1]}"
+            edge_key_2 = f"{pairs[i][1]}--{pairs[i][0]}"
+            if edge_key_1 in self.pairs_to_edge_ids:
+                cached_pairs.append(edge_key_1)
+                del pairs[i]
+            elif edge_key_2 in self.pairs_to_edge_ids:
+                cached_pairs.append(edge_key_2)
+                del pairs[i]
+            else:
+                i += 1
+
+        url = self.plover_url + "/get_edges"
+        query = {"pairs": pairs}
         try:
-            response = requests.post(self.plover_url + endpoint, headers={'accept': 'application/json'}, json=data)
-            json = response.json()
-            self.edges[edge_key_1] = json
-            return json
+            knowledge_graph = {'edges': {}, 'nodes': {}}
+            if len(pairs) != 0:
+                response = requests.post(url, headers={'accept': 'application/json'}, json=query)
+                json = response.json()
+                self.pairs_to_edge_ids.update(json["pairs_to_edge_ids"])
+                knowledge_graph = json["knowledge_graph"]
+                self.knowledge_graph['edges'].update(knowledge_graph['edges'])
+                self.knowledge_graph['nodes'].update(knowledge_graph['nodes'])
+            for cached_pair in cached_pairs:
+                list_of_edges = self.pairs_to_edge_ids[cached_pair]
+                cached_nodes = cached_pair.split("--")
+                if cached_nodes[0] in self.knowledge_graph['nodes']:
+                    knowledge_graph['nodes'][cached_nodes[0]] = self.knowledge_graph['nodes'][cached_nodes[0]]
+                if cached_nodes[1] in self.knowledge_graph['nodes']:
+                    knowledge_graph['nodes'][cached_nodes[1]] = self.knowledge_graph['nodes'][cached_nodes[1]]
+                for edge in list_of_edges:
+                    knowledge_graph['edges'][edge] = self.knowledge_graph['edges'][edge]
+            return knowledge_graph
         except Exception as e:
-            arax_response.warning(f"Cannot retrieve {data} from plover DB with error: {e}")
+            arax_response.warning(f"Cannot retrieve {query} from plover DB with url: {url}, Error: {e}")
             return None

code/ARAX/ARAXQuery/Path_Finder/converter/GraphToKnowledgeGraphConverter.py

@@ -6,9 +6,8 @@
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
 from converter.PathListToGraphConverter import PathListToGraphConverter
-from converter.SimpleGraphToContentGraphConverter import SimpleGraphToContentGraphConverter
-from converter.EdgeExtractorFromPloverDB import EdgeExtractorFromPloverDB
-
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../../../../UI/OpenAPI/python-flask-server/")
+from openapi_server.models.knowledge_graph import KnowledgeGraph
 
 class GraphToKnowledgeGraphConverter:
 
@@ -25,17 +24,10 @@ def __init__(
     def convert(self, response, paths):
         nodes, edges = PathListToGraphConverter(self.qnode_1_id, self.qnode_2_id).convert(paths)
 
-        nodes, edges = SimpleGraphToContentGraphConverter(
-            self.edge_extractor
-        ).convert(nodes, edges, response)
+        pairs = []
+        for _, edge in edges.items():
+            pairs.append([nodes[edge[0]], nodes[edge[1]]])
+
+        knowledge_graph = self.edge_extractor.get_edges(pairs, response)
 
-        qg_organized_knowledge_graph = (
-            KG2Querier(response, self.edge_extractor.get_extractor_url())._load_plover_answer_into_object_model(
-                {
-                    "nodes": nodes,
-                    "edges": edges
-                },
-                response
-            )
-        )
-        return eu.convert_qg_organized_kg_to_standard_kg(qg_organized_knowledge_graph)
+        return KnowledgeGraph().from_dict(knowledge_graph)

code/ARAX/ARAXQuery/Path_Finder/converter/HugeGraphConverter.py

@@ -51,11 +51,10 @@ def convert(self, response):
             self.qnode_2_id,
             self.edge_extractor).convert(response, self.paths)
 
-        essence = ''
         if self.node_in_between_id in knowledge_graph_src_dest.nodes:
             essence = knowledge_graph_src_dest.nodes[self.node_in_between_id].name
         else:
-            response.warning("Could not find main node id name to fill essence variable!")
+            response.warning(f"Could not find main node id name: {self.node_in_between_id} to fill essence variable!")
             return
 
         if (self.node_category_constraint != ''

code/ARAX/ARAXQuery/Path_Finder/repo/PloverDBRepo.py

@@ -16,31 +16,14 @@ def __init__(self, plover_url, degree_repo=NodeDegreeRepo()):
         self.degree_repo = degree_repo
 
     def get_neighbors(self, node, limit=-1):
-        endpoint = "/query"
-        data = {
-            "edges": {
-                "e00": {
-                    "subject": "n00",
-                    "object": "n01"
-                }
-            },
-            "nodes": {
-                "n00": {
-                    "ids": [node.id]
-                },
-                "n01": {
-                    "categories": ["biolink:NamedThing"]
-                }
-            },
-            "include_metadata": False,
-            "respect_predicate_symmetry": True
-        }
+        endpoint = "/get_neighbors"
+        data = {"node_ids": [node.id], "categories": ["biolink:NamedThing"]}
         result = []
         try:
             response = requests.post(self.plover_url + endpoint, headers={'accept': 'application/json'}, json=data)
             response.raise_for_status()
             json = response.json()
-            result = [Node(i) for i in json['nodes']['n01']]
+            result = [Node(i) for i in json[node.id]]
         except requests.exceptions.RequestException as e:
             # log here print(f"Request error: {e}")
             pass

code/UI/OpenAPI/specifications/export/KG2/1.5.0/openapi.yaml

@@ -51,7 +51,7 @@ servers:
   url: https://kg2.ci.transltr.io/api/rtxkg2/v1.4
   x-maturity: staging
 - description: RTX-KG2 TRAPI 1.5 endpoint - development
-  url: https://kg2cplover3.rtx.ai:9990
+  url: https://kg2cploverdb.ci.transltr.io
   x-maturity: development
 tags:
 - description: Retrieve the meta knowledge graph representation of this TRAPI web

code/config_dbs.json

@@ -14,7 +14,7 @@
       "xcrg_decrease_model": "/translator/data/orangeboard/databases/KG2.10.0/xcrg_decrease_model_v1.0.KG2.10.0.pt"
    },
    "plover": {
-      "dev": "https://kg2cplover3.rtx.ai:9990",
+      "dev": "https://kg2cploverdb.ci.transltr.io",
       "test": "https://kg2cploverdb.test.transltr.io",
       "prod": "https://kg2cploverdb.transltr.io"
    },