Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

major refactor of TRAPI and summarization code #231

Open
wants to merge 74 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
354e57e
cleanup summary edge abstraction; other minor cleanups/comments
May 8, 2024
111946e
cleanup and begin splitting up trapi.mjs
May 15, 2024
a687846
Merge branch 'develop' into refactor/trapi-cleanup
May 15, 2024
d04521d
refactor TRAPI query graph generation
May 17, 2024
78aae5f
add TRAPI loading into QGraph elements
May 19, 2024
566d111
fix merge conflicts
May 21, 2024
9a003d6
fix bug when merging publications
May 21, 2024
4fa3528
add qualified predicate to MVP2 query
May 21, 2024
fcaf428
fix merge conflict
May 22, 2024
51a8346
Merge branch 'develop' into refactor/trapi-cleanup
May 23, 2024
22ec3a6
reorganize file
May 23, 2024
c4676c8
convert summary metadata to class
May 23, 2024
034b2b8
simplify the number of extraction rule helpers
May 24, 2024
a2e4178
fix merge conflict
May 24, 2024
5a6a9e4
add transfromProperty
May 24, 2024
47ae158
fix bug in transformProperty
May 24, 2024
5c9c357
fix bug with getProperty
May 24, 2024
c59f163
changes to attribute gathering for clarity
May 29, 2024
a6aab0e
finish cleaning up attribute extraction code
May 30, 2024
721523e
Merge branch 'develop' into refactor/trapi-cleanup
May 31, 2024
1049abe
fix merge conflict
Jun 11, 2024
acd8e7a
add TODO
Jun 11, 2024
9b954a8
change name of summarization lib
Jun 13, 2024
b57cf2a
catch branch up with main (manual merge)
Jun 13, 2024
d2537e7
add TRAPI helpers for getting code kedge properties
Jun 14, 2024
3789b5b
add binding -> graph element helpers in TRAPI library
Jun 14, 2024
9fdd77c
simplify endpoint finding
Jun 14, 2024
f2bf832
add comments for getting result endpoints
Jun 14, 2024
40a9c48
add qualifier and support graph helpers
Jun 15, 2024
36736c1
cleanup qualified predicate generation
Jun 15, 2024
128214a
fix bug in getting tags
Jun 15, 2024
83659d8
remove extra semicolon
Jun 15, 2024
ea9eb42
fix bug that caused support paths not to be generated
Jun 17, 2024
ca5bbc3
better naming for qualified predicate generation
Jun 18, 2024
59cd5a3
improve tagging code
Jun 19, 2024
9985f7e
fix bug in tagging attributes introduced in refactor
Jun 20, 2024
01fe1e7
remove debug log files
Jun 20, 2024
a7fecbb
merge develop with refactor; fix regression test structure
Jul 4, 2024
bbc107a
doc updates; get rid of uneeded getter function
Jul 8, 2024
524b6c9
refactor summary edges and dependant code
Jul 8, 2024
e5d44a9
update to develop; fix regression testing
Jul 12, 2024
6dc99d9
Merge branch 'develop' into refactor/trapi-cleanup
Jul 19, 2024
a02efcc
Throw an explicit error when a message is missing a QG
Jul 19, 2024
8835ec6
Shorten variable names for summary and fragment
Jul 19, 2024
6561160
Change naming for tag functions for consistency
Jul 19, 2024
89bf1aa
Move helpers to trapi.mjs; Rename ids for clarity
Jul 19, 2024
97c9a5b
Shorten names to increase readability
Jul 19, 2024
55217dd
Move Summary classes; Shorten names for clarity
Jul 19, 2024
bc0deb5
Add utility functions; Rename for clarity
Jul 19, 2024
9853989
Renaming for clarity
Jul 20, 2024
35aba70
Naming for clarity
Jul 20, 2024
108eb4b
Fix regression tests
Jul 20, 2024
91e67ee
fix bugs in bta processing
Jul 22, 2024
c79189e
resolve equal summary paths in pidSort
Jul 22, 2024
020bb16
Fix bug with result tag gathering
Jul 22, 2024
e8c1d2f
fix merge conflicts
Jul 23, 2024
a772f63
Fix multiple bugs found in regression testing
Jul 23, 2024
d630bfd
Fix bug in pidSort
Jul 24, 2024
4156ca0
Merge branch 'develop' into refactor/trapi-cleanup
Jul 24, 2024
96fd38b
fix merge conflicts with test/regression
Jul 24, 2024
dec31db
remove unnecessary log
Jul 24, 2024
78bf029
fix biothings curie annotation gathering
Jul 25, 2024
fa511ae
Change regression configuration to be production config
Jul 25, 2024
04f1540
fix bugs with BTA attribuate process
Jul 29, 2024
1e50003
Merge branch 'develop' into refactor/trapi-cleanup
Jul 30, 2024
63b2145
Merge branch 'develop' into refactor/trapi-cleanup
Jul 30, 2024
b666d5d
Merge branch 'develop' into refactor/trapi-cleanup
Jul 31, 2024
acdda0b
Remove bad regression test
Jul 31, 2024
97dd1e7
merge with develop
Sep 19, 2024
126ade0
finish tag restructuring merge
Sep 20, 2024
7e0ca26
Add submission of pathfinder queries
Sep 23, 2024
a59da2f
Finish merging pathfinder changes
Sep 23, 2024
02c2437
fix bug with wrong name for path id generation
Sep 23, 2024
b347cf4
add constraints to pathfinder query
Sep 24, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions StartServer.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { bootstrapConfig } from './lib/config.mjs';

import { loadBiolink } from './lib/biolink-model.mjs';
import { loadChebi } from './lib/chebi.mjs';
import { loadTrapi } from './lib/trapi.mjs';
import { TranslatorService } from './services/TranslatorService.mjs';
import { TranslatorServicexFEAdapter } from './adapters/TranslatorServicexFEAdapter.mjs';
import { ARSClient } from './lib/ARSClient.mjs';
Expand Down Expand Up @@ -33,10 +34,8 @@ const SERVER_CONFIG = await (async function() {
return bootstrapConfig(basefile, overrides);
})();

await loadBiolink(SERVER_CONFIG.biolink.version,
SERVER_CONFIG.biolink.support_deprecated_predicates,
SERVER_CONFIG.biolink.infores_catalog,
SERVER_CONFIG.biolink.prefix_catalog);
await loadBiolink(SERVER_CONFIG.biolink);
await loadTrapi(SERVER_CONFIG.trapi);
await loadChebi();

// Bootstrap the translator service.
Expand Down
8 changes: 4 additions & 4 deletions adapters/TranslatorServicexFEAdapter.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict';

import * as arsmsg from '../lib/ARSMessages.mjs';
import * as trapi from '../lib/trapi.mjs';
import * as sm from '../lib/summarization.mjs';

/* Translate messages coming from the Translator Service into the formats that the Frontend (FE) app expects */
/* This module should not contain logic that goes beyond message transformations */
Expand Down Expand Up @@ -49,15 +49,15 @@ class TranslatorServicexFEAdapter {
}
});

const summary = await trapi.creativeAnswersToSummary(
const smry = await sm.answersToSmry(
msg.pk,
data,
maxHops);
summary.meta.timestamp = msg.meta.timestamp;
smry.meta.timestamp = msg.meta.timestamp;

return {
status: determineStatus(msg),
data: summary
data: smry
};
}
}
Expand Down
4 changes: 4 additions & 0 deletions configurations/ci.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
"port": 8386,
"response_timeout": 300,
"json_payload_limit": "20mb",
"trapi": {
"query_subject_key": "sn",
"query_object_key": "on"
},
"ars_endpoint": {
"host": "ars.ci.transltr.io",
"post_uri": "/ars/api/submit",
Expand Down
4 changes: 4 additions & 0 deletions configurations/production.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
"port": 8386,
"response_timeout": 300,
"json_payload_limit": "1mb",
"trapi": {
"query_subject_key": "sn",
"query_object_key": "on"
},
"ars_endpoint": {
"host": "ars-prod.transltr.io",
"post_uri": "/ars/api/submit",
Expand Down
4 changes: 4 additions & 0 deletions configurations/test.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
"port": 8386,
"response_timeout": 300,
"json_payload_limit": "1mb",
"trapi": {
"query_subject_key": "sn",
"query_object_key": "on"
},
"ars_endpoint": {
"host": "ars.test.transltr.io",
"post_uri": "/ars/api/submit",
Expand Down
90 changes: 47 additions & 43 deletions lib/biolink-model.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,54 +9,58 @@ let DEPRECATED_TO_QUALIFIED_PREDICATE_MAP = null;
let PREFIX_CATALOG = null;
let PREFIX_EXCLUDE_LIST = null;

export async function loadBiolink(biolinkVersion, supportDeprecatedPredicates, inforesCatalog, prefixCatalog) {
export async function loadBiolink(biolinkConfig) {
const biolinkVersion = biolinkConfig.version;
const supportDeprecatedPreds = biolinkConfig.support_deprecated_predicates;
const inforesCatalog = biolinkConfig.infores_catalog;
const prefixCatalog = biolinkConfig.prefix_catalog;
const biolinkModel = await cmn.readJson(`./assets/biolink-model/${biolinkVersion}/biolink-model.json`);
const slots = cmn.jsonGet(biolinkModel, 'slots');
const classes = cmn.jsonGet(biolinkModel, 'classes');
BIOLINK_PREDICATES = makeBlPredicates(slots);
BIOLINK_PREDICATES = makeBlPreds(slots);
BIOLINK_CLASSES = makeBlClasses(classes);
INFORES_CATALOG = await cmn.readJson(`./assets/biolink-model/common/${inforesCatalog}`);
PREFIX_CATALOG = await cmn.readJson(`./assets/biolink-model/common/${prefixCatalog.path}`);
PREFIX_EXCLUDE_LIST = prefixCatalog.exclude;
if (supportDeprecatedPredicates) {
if (supportDeprecatedPreds) {
DEPRECATED_TO_QUALIFIED_PREDICATE_MAP = await cmn.readJson(`./assets/biolink-model/${biolinkVersion}/deprecated-predicate-mapping.json`);
}
}

export function tagBiolink(str) {
return biolinkifyPredicate(str);
return biolinkifyPred(str);
}

export function isDeprecatedPredicate(s) {
export function isDeprecatedPred(s) {
return !!DEPRECATED_TO_QUALIFIED_PREDICATE_MAP &&
DEPRECATED_TO_QUALIFIED_PREDICATE_MAP[sanitizeBiolinkItem(s)] !== undefined;
}

export function isBiolinkPredicate(s) {
const sanitizedPredicate = sanitizeBiolinkItem(s);
return BIOLINK_PREDICATES[sanitizedPredicate] !== undefined ||
isDeprecatedPredicate(sanitizedPredicate);
export function isBiolinkPred(s) {
const sanitizedPred = sanitizeBiolinkItem(s);
return BIOLINK_PREDICATES[sanitizedPred] !== undefined ||
isDeprecatedPred(sanitizedPred);
}

export function sanitizeBiolinkItem(pred) {
return pred.replaceAll('_', ' ').replaceAll('biolink:', '');
}

export function invertBiolinkPredicate(pred, biolinkify = false) {
export function invertBiolinkPred(pred, biolinkify = false) {
const p = sanitizeBiolinkItem(pred);
const biolinkPredicate = cmn.jsonGet(BIOLINK_PREDICATES, p, false);
if (biolinkPredicate) {
const biolinkPred= cmn.jsonGet(BIOLINK_PREDICATES, p, false);
if (biolinkPred) {
if (biolinkify) {
return biolinkifyPredicate(biolinkPredicate.inverse);
return biolinkifyPred(biolinkPred.inverse);
}
return biolinkPredicate.inverse;
return biolinkPred.inverse;
}
throw InvalidPredicateError(p);
}

export function deprecatedPredicateToPredicateAndQualifiers(predicate) {
const qualifiedPredicate = DEPRECATED_TO_QUALIFIED_PREDICATE_MAP[predicate];
return [qualifiedPredicate.predicate, qualifiedPredicate];
export function deprecatedPredToPredAndQualifiers(pred) {
const qualifiedPred = DEPRECATED_TO_QUALIFIED_PREDICATE_MAP[pred];
return [qualifiedPred.predicate, qualifiedPred];
}

export function inforesToProvenance(infores) {
Expand Down Expand Up @@ -117,13 +121,13 @@ export function curieToUrl(curie) {
return `${url}${curieId}`;
}

export function predicateToUrl(predicate) {
const p = predicate.replaceAll(' ', '_');
export function predToUrl(pred) {
const p = pred.replaceAll(' ', '_');
return `https://biolink.github.io/biolink-model/${p}/`
}

function InvalidPredicateError(predicate) {
const error = new Error(`Expected a valid biolink predicate. Got: ${predicate}`, 'biolink-model.mjs');
function InvalidPredicateError(pred) {
const error = new Error(`Expected a valid biolink predicate. Got: ${pred}`, 'biolink-model.mjs');
}
InvalidPredicateError.prototype = Object.create(Error.prototype);

Expand Down Expand Up @@ -164,39 +168,39 @@ function getInverse(pred, record) {
return cmn.jsonGet(record, 'inverse', false);
}

function distanceFromRelatedTo(slots, predicate) {
function distanceFromRelatedTo(slots, pred) {
for (let level = 0; ;level += 1) {
if (predicate === 'related to') {
if (pred === 'related to') {
return level;
}

let predicateObj = cmn.jsonGet(slots, predicate, false);
if (!predicateObj) {
let predObj = cmn.jsonGet(slots, pred, false);
if (!predObj) {
return -1;
}

predicate = cmn.jsonGet(predicateObj, 'is_a', false);
pred = cmn.jsonGet(predObj, 'is_a', false);
}
}

function makeBlPredicate(predicate, record, rank) {
function makeBlPred(pred, record, rank) {
return {
'parent': getParent(predicate, record),
'isCanonical': isCanonical(predicate, record),
'isSymmetric': isSymmetric(predicate, record),
'isDeprecated': isDeprecated(predicate, record),
'inverse': getInverse(predicate, record),
'parent': getParent(pred, record),
'isCanonical': isCanonical(pred, record),
'isSymmetric': isSymmetric(pred, record),
'isDeprecated': isDeprecated(pred, record),
'inverse': getInverse(pred, record),
'rank': rank,
};
}

function makeBlPredicates(slots) {
function makeBlPreds(slots) {
let blPreds = {};
Object.keys(slots).forEach((pred) => {
const rank = distanceFromRelatedTo(slots, pred);
if (rank >= 0) {
const record = slots[pred];
blPreds[pred] = makeBlPredicate(pred, record, rank);
blPreds[pred] = makeBlPred(pred, record, rank);
}
});

Expand All @@ -213,7 +217,7 @@ function makeBlPredicates(slots) {
return blPreds;
}

function biolinkifyPredicate(pred) {
function biolinkifyPred(pred) {
let s = pred.replaceAll(' ', '_');
if (s.startsWith('biolink:')) {
return s;
Expand All @@ -222,29 +226,29 @@ function biolinkifyPredicate(pred) {
return `biolink:${s}`;
}

function getBiolinkPredicateData(pred) {
function getBiolinkPredData(pred) {
return cmn.jsonGet(BIOLINK_PREDICATES, sanitizeBiolinkItem(pred), false);
}

function isBiolinkPredicateMoreSpecific(pred1, pred2) {
function isBiolinkPredMoreSpecific(pred1, pred2) {
const p1 = sanitizeBiolinkItem(pred1);
const p2 = sanitizeBiolinkItem(pred2);
const biolinkPredicate1 = getBiolinkPredicateData(p1);
const biolinkPredicate2 = getBiolinkPredicateData(p2);
const biolinkPred1 = getBiolinkPredData(p1);
const biolinkPred2 = getBiolinkPredData(p2);

if (!biolinkPredicate1) {
if (!biolinkPred1) {
throw InvalidPredicateError(p1);
}
else if (!biolinkPredicate2) {
else if (!biolinkPred2) {
throw InvalidPredicateError(p2);
}
else {
return pred1.rank > pred2.rank;
}
}

function sortBiolinkPredicates(preds) {
return sort(pres, isBiolinkPredicateMoreSpecific);
function sortBiolinkPreds(preds) {
return sort(pres, isBiolinkPredMoreSpecific);
}

/* Biolink category-related functions
Expand Down
35 changes: 18 additions & 17 deletions lib/biothings-annotation.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import * as chebi from './chebi.mjs';
export function getFdaApproval(annotation) {
return parseAnnotation(
annotation,
noHandler,
genDefaultHandler(),
getChemicalFdaApproval,
noHandler);
genDefaultHandler());
}

export function getDescription(annotation) {
Expand All @@ -22,37 +22,40 @@ export function getDescription(annotation) {
export function getChebiRoles(annotation) {
return parseAnnotation(
annotation,
noHandler,
genDefaultHandler(),
getChemicalChebiRoles,
noHandler);
genDefaultHandler());
}

export function getDrugIndications(annotation) {
const defaultHandler = genDefaultHandler([]);
return parseAnnotation(
annotation,
noHandler,
defaultHandler,
getChemicalDrugIndications,
noHandler);
defaultHandler,
defaultHandler);
}

export function getCuries(annotation) {
const defaultHandler = genDefaultHandler([]);
return parseAnnotation(
annotation,
getDiseaseMeshCuries,
noHandler,
noHandler);
defaultHandler,
defaultHandler,
defaultHandler);
}

export function getNames(annotation) {
return parseAnnotation(
annotation,
noHandler,
genDefaultHandler(),
getChemicalNames,
noHandler);
genDefaultHandler());
}

function parseAnnotation(annotation, diseaseHandler, chemicalHandler, geneHandler, fallback = null) {
annotation = annotation[0];
function parseAnnotation(annotation, diseaseHandler, chemicalHandler, geneHandler, defaultHandler = genDefaultHandler()) {
if (isDisease(annotation)) {
return diseaseHandler(annotation);
}
Expand All @@ -65,7 +68,7 @@ function parseAnnotation(annotation, diseaseHandler, chemicalHandler, geneHandle
return geneHandler(annotation);
}

return fallback;
return defaultHandler();
}

function getDiseaseDescription(annotation) {
Expand All @@ -82,15 +85,13 @@ function getDiseaseMeshCuries(annotation) {
['mondo', 'xrefs', 'mesh'],
['disease_ontology', 'xrefs', 'mesh']
];

const curies = [];
for (const path of paths) {
const curie = cmn.jsonGetFromKpath(annotation, path, null);
if (curie) {
curies.push(`MESH:${curie}`);
}
}

return curies;
}

Expand Down Expand Up @@ -181,6 +182,6 @@ function isGene(annotation) {
return annotation.symbol !== undefined;
}

function noHandler(annotation) {
return null;
function genDefaultHandler(fallback = null) {
return function() { return fallback };
}
Loading