Xqtl (#1852)

* Attempt to infer file format from contents when extension isn't recognized

dev/qtl/xqtl.html

@@ -69,15 +69,17 @@ <h3>QTL example</h3>
             {
                 type: "qtl",
                 format: "qtl",
-                name: "Moncyte mQTL",
+
+                name: "Monocyte mQTL",
                 url: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_mQTL.tsv.gz",
                 indexURL: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_mQTL.tsv.gz.tbi",
                 visibilityWindow: 4000000
             },
             {
                 type: "qtl",
                 format: "qtl",
-                name: "Moncyte hQTL",
+                name: "Monocyte hQTL",
+
                 url: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_hQTL.tsv.gz",
                 indexURL: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_hQTL.tsv.gz.tbi",
                 visibilityWindow: 4000000

examples/xqtl.html

@@ -65,15 +65,17 @@ <h3>QTL example</h3>
             {
                 type: "qtl",
                 format: "qtl",
-                name: "Moncyte mQTL",
+
+                name: "Monocyte mQTL",
                 url: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_mQTL.tsv.gz",
                 indexURL: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_mQTL.tsv.gz.tbi",
                 visibilityWindow: 4000000
             },
             {
                 type: "qtl",
                 format: "qtl",
-                name: "Moncyte hQTL",
+
+                name: "Monocyte hQTL",
                 url: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_hQTL.tsv.gz",
                 indexURL: "https://igv-genepattern-org.s3.amazonaws.com/test/qtl/Monocyte_hQTL.tsv.gz.tbi",
                 visibilityWindow: 4000000

js/bam/bamSource.js

@@ -30,7 +30,6 @@ import BamWebserviceReader from "./bamWebserviceReader.js"
 import HtsgetBamReader from "../htsget/htsgetBamReader.js"
 import CramReader from "../cram/cramReader.js"
 import {isDataURL} from "../util/igvUtils.js"
-import * as TrackUtils from "../util/trackUtils.js"
 import {StringUtils} from "../../node_modules/igv-utils/src/index.js"
 
 class BamSource {
@@ -59,7 +58,7 @@ class BamSource {
         } else {
             if (!this.config.indexURL && config.indexed !== false) {
                 if (StringUtils.isString(this.config.url)) {
-                    const inferIndexPath = TrackUtils.inferIndexPath(this.config.url, "bai")
+                    const inferIndexPath = inferIndexPath(this.config.url, "bai")
                     if (inferIndexPath) {
                         console.error(`Warning: no indexURL specified for ${this.config.url}.  Guessing ${inferIndexPath}`)
                         this.config.indexURL = inferIndexPath

js/browser.js

@@ -17,7 +17,7 @@ import GenomeUtils from "./genome/genomeUtils.js"
 import ReferenceFrame, {createReferenceFrameList} from "./referenceFrame.js"
 import {createColumn, doAutoscale, getElementAbsoluteHeight, getFilename} from "./util/igvUtils.js"
 import {createViewport} from "./util/viewportUtils.js"
-import {defaultSequenceTrackOrder} from './sequenceTrack.js'
+import {bppSequenceThreshold, defaultSequenceTrackOrder} from './sequenceTrack.js'
 import version from "./version.js"
 import FeatureSource from "./feature/featureSource.js"
 import {defaultNucleotideColors} from "./util/nucleotideColors.js"
@@ -59,11 +59,11 @@ import Genome from "./genome/genome.js"
 import {setDefaults} from "./igv-create.js"
 import {trackViewportPopoverList} from './trackViewport.js'
 import TrackBase from "./trackBase.js"
-import {bppSequenceThreshold} from "./sequenceTrack.js"
 import {loadGenbank} from "./gbk/genbankParser.js"
 import igvCss from "./embedCss.js"
 import {sampleInfoTileWidth, sampleInfoTileXShim} from "./sample/sampleInfoConstants.js"
 import QTLSelections from "./qtl/qtlSelections.js"
+import {inferFileFormat, inferFileFormatFromContents} from "./util/fileFormatUtils.js"
 
 
 // css - $igv-scrollbar-outer-width: 14px;
@@ -1206,15 +1206,8 @@ class Browser {
             } else if (config.fastaURL) {
                 config.format = "fasta"  // by definition
             } else {
-                let filename = config.filename
-                if (!filename) {
-                    filename = await getFilename(url)
-                }
-
-                const format = TrackUtils.inferFileFormat(filename)
-                if ("tsv" === format) {
-                    config.format = await TrackUtils.inferFileFormatFromHeader(config)
-                } else if (format) {
+                const format = await inferFileFormat(config)
+                if (format) {
                     config.format = format
                 } else {
                     if (config.sourceType === "htsget") {

js/feature/featureFileReader.js

@@ -172,16 +172,27 @@ class FeatureFileReader {
             }
 
         } else {
-            // If this is a non-indexed file we will load all features in advance
-            const options = buildOptions(this.config)
-            let data = await igvxhr.loadByteArray(this.config.url, options)
+
+            let data
+
+            if (this.config._filecontents) {
+                // In rare instances the entire file must be read and decoded to determine the file format.
+                // When this occurs the file contents are temporarily stashed to prevent needing to read the file twice
+                data = this.config._filecontents
+                delete this.config._filecontents
+            } else {
+                // If this is a non-indexed file we will load all features in advance
+                const options = buildOptions(this.config)
+                data = await igvxhr.loadByteArray(this.config.url, options)
+            }
 
             // If the data size is < max string length decode entire string with TextDecoder.  This is much faster
             // than decoding by line
             if (data.length < MAX_STRING_LENGTH) {
                 data = new TextDecoder().decode(data)
             }
 
+
             let dataWrapper = getDataWrapper(data)
             this.header = await this.parser.parseHeader(dataWrapper)
 

js/feature/featureParser.js

@@ -23,22 +23,21 @@
  * THE SOFTWARE.
  */
 
-import * as TrackUtils from '../util/trackUtils.js'
 import {decodeBedpe, decodeBedpeDomain, fixBedPE} from './decode/bedpe.js'
 import {decodeInteract} from "./decode/interact.js"
 import {
     decodeBed,
     decodeBedGraph,
+    decodeBedmethyl,
+    decodeGappedPeak,
     decodeGenePred,
     decodeGenePredExt,
+    decodeNarrowPeak,
     decodePeak,
     decodeReflat,
     decodeRepeatMasker,
     decodeSNP,
-    decodeWig,
-    decodeBedmethyl,
-    decodeGappedPeak,
-    decodeNarrowPeak
+    decodeWig
 } from "./decode/ucsc.js"
 import {decodeGFF3, decodeGTF} from "./gff/gff.js"
 import {decodeFusionJuncSpan} from "./decode/fusionJuncSpan.js"
@@ -48,6 +47,8 @@ import {decodeGcnv} from "../gcnv/gcnvDecoder.js"
 import DecodeError from "./decode/decodeError.js"
 import GFFHelper from "./gff/gffHelper.js"
 
+import {getFormat} from "../util/fileFormats.js"
+
 /**
  *  Parser for column style (tab delimited, etc) text file formats (bed, gff, vcf, etc).
  *
@@ -319,7 +320,7 @@ class FeatureParser {
                 this.delimiter = "\t"
                 break
             default:
-                const customFormat = TrackUtils.getFormat(format)
+                const customFormat = getFormat(format)
                 if (customFormat !== undefined) {
                     this.decode = decodeCustom
                     this.header.customFormat = customFormat

js/gwas/gwasParser.js

@@ -99,6 +99,29 @@ class GWASParser {
         }
         return allFeatures
     }
+
+    /**
+     * Test first line to see if this is a GWAS file.  Used to determine file format for the case of generic
+     * extensions such as "tsv"
+     * @param firstLine
+     */
+    static isGWAS(firstLine) {
+        const tokens = firstLine.split('\t')
+        if (tokens.length < 5) {
+            return false
+        }
+        const requiredHeaders =
+            [
+                ['chr', 'chromosome', 'chr_id', 'chrom'],
+                ['bp', 'pos', 'position', 'chr_pos', 'chromEnd'],
+                ['p', 'pval', 'p-value', 'p.value']
+            ]
+        for (let h of requiredHeaders) {
+            if (!tokens.some(t => h.includes(t.toLowerCase()))) return false
+        }
+        return true
+    }
+
 }
 
 class GWASFeature {

js/index.js

@@ -7,13 +7,14 @@ import {createBrowser, createTrack, removeAllBrowsers, removeBrowser, visibility
 import embedCss from "./embedCss.js"
 import version from "./version.js"
 import * as TrackUtils from "./util/trackUtils.js"
-import {registerFileFormats} from "./util/trackUtils.js"
 import {igvxhr} from "../node_modules/igv-utils/src/index.js"
 import {registerTrackClass, registerTrackCreatorFunction} from "./trackFactory.js"
 import TrackBase from "./trackBase.js"
 import Hub from "./ucsc/ucscHub.js"
 import Browser from "./browser.js"
 
+import {registerFileFormats} from "./util/fileFormats.js"
+
 const setApiKey = igvxhr.setApiKey
 
 function setGoogleOauthToken(accessToken) {

js/qtl/qtlParser.js

@@ -78,7 +78,7 @@ class QTLParser {
     async parseHeader(dataWrapper) {
 
         const config = this.config
-        if(config.delimiter) this.delimiter = config.delimiter
+        if (config.delimiter) this.delimiter = config.delimiter
 
         const headerLine = await dataWrapper.nextLine()
         const columns = this.parseHeaderLine(headerLine)
@@ -180,6 +180,30 @@ class QTLParser {
         }
         return allFeatures
     }
+
+    /**
+     * Test first line to see if this is a QTL file.  Used to determine file format for the case of generic
+     * extensions such as "tsv"
+     * @param firstLine
+     */
+    static isQTL(firstLine) {
+        const tokens = firstLine.split('\t')
+        if (tokens.length < 5) {
+            return false
+        }
+        const requiredHeaders =
+            [
+                ['chr', 'chromosome', 'chr_id', 'chrom'],
+                ['bp', 'pos', 'position', 'chr_pos', 'chromEnd'],
+                ['p', 'pval', 'p-value', 'p.value'],
+                ['rsid', 'variant', 'snp'],
+                ['phenotype', 'gene', 'gene_id', 'molecular_trait_id']
+            ]
+        for (let h of requiredHeaders) {
+            if (!tokens.some(t => h.includes(t.toLowerCase()))) return false
+        }
+        return true
+    }
 }
 
 

js/roi/ROISet.js

@@ -2,7 +2,7 @@ import {FileUtils, StringUtils} from '../../node_modules/igv-utils/src/index.js'
 import FeatureSource from '../feature/featureSource.js'
 import {appleCrayonRGBA} from '../util/colorPalletes.js'
 import {computeWGFeatures} from "../feature/featureUtils.js"
-import * as TrackUtils from "../util/trackUtils.js"
+import {inferFileFormatFromName} from "../util/fileFormatUtils.js"
 
 
 const appleCrayonColorName = 'nickel'
@@ -38,7 +38,7 @@ class ROISet {
                 config.format = config.format.toLowerCase()
             } else {
                 const filename = FileUtils.getFilename(config.url)
-                config.format = TrackUtils.inferFileFormat(filename)
+                config.format = inferFileFormatFromName(filename)
             }
             this.featureSource = config.featureSource || FeatureSource(config, genome)
         }