diff --git a/.gitignore b/.gitignore index d7085efe..4e4cf32b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ data/credentials.json node_modules/ sprites.hbs temp.js +temp.json working.js \ No newline at end of file diff --git a/data/DataManager.js b/data/DataManager.js index 04501193..62640999 100644 --- a/data/DataManager.js +++ b/data/DataManager.js @@ -6,7 +6,7 @@ import { outputFile } from 'fs-extra/esm' import path from 'node:path' import ProgressBar from 'progress' import { readFile } from 'node:fs/promises' -import { setTimeout } from 'node:timers/promises' +import retryRequest from './utilities/retryRequest.js' import stringifyCSV from './utilities/stringifyCSV.js' export default class DataManager { @@ -99,29 +99,8 @@ export default class DataManager { }) for (const key of this.languages.keys()) { - - let waitTime = 1 - - const makeRequest = async () => { - try { - await this.fetchLanguageComponents(key) - } catch (e) { - if (e?.response?.status === 429) { - - waitTime *= 2 - console.warn(`\nHit rate limit. Retrying after ${ waitTime }ms.`) - await setTimeout(waitTime) - await makeRequest() - - } else { - throw e - } - } - } - - await makeRequest() // Kick off request sequence + await retryRequest(this.fetchLanguageComponents.bind(this), key) progressBar.tick() - } } diff --git a/data/csv/Abenaki/components.csv b/data/csv/Abenaki/components.csv index 028dcaf7..a34d2fb2 100644 --- a/data/csv/Abenaki/components.csv +++ b/data/csv/Abenaki/components.csv @@ -1,4 +1,4 @@ -ID,Author Surname,Source Code,Page #,ISO code,Dialect,Glottocode,Form (original orthography),Orthography Key Code,Form (project orthography),UR (if given; if different),UR (project orthography),PA form (original orthography),PA form (project orthography),Translation,Project Definition,Component Type,Subcategory,Initial: Reduplicated,Final: concrete / abstract,Final: secondary (y/n/b),Base Category (if secondary)-A,Match AI,Match II,Match TA,Match TI,Allomorph-A,Condition-A,Component occurs in stem (orig. orth)-A,Component occurs in stem (proj. orth)-A,Stem UR-A,Stem category-A,Stem subcategory-A,Stem translation-A,Secondary stem-A,Stem Source-A,Formative/component occurs in what component(s)-A,Contains component (enter)-A,Contains formative (enter)-A,Deverbal (y/n),Deverbal from,Comments,Speaker,1st check done,2nd check done +ID,Author Surname,Source Code,Page #,ISO code,Dialect,Glottocode,Form (original orthography),Orthography Key Code,Form (project orthography),UR (if given; if different),UR (project orthography),PA form (original orthography),PA form (project orthography),Translation,Tags,Component Type,Subcategory,Initial: Reduplicated,Final: concrete / abstract,Final: secondary (y/n/b),Base Category (if secondary)-A,Match AI,Match II,Match TA,Match TI,Allomorph-A,Condition-A,Component occurs in stem (orig. orth)-A,Component occurs in stem (proj. orth)-A,Stem UR-A,Stem category-A,Stem subcategory-A,Stem translation-A,Secondary stem-A,Stem Source-A,Formative/component occurs in what component(s)-A,Contains component (enter)-A,Contains formative (enter)-A,Deverbal (y/n),Deverbal from,Comments,Speaker,1st check done,2nd check done 1,Goddard,IG1965,0213,,,aben1250,ôben-,AB:2,,,,*a·p-,,untie,untie,initial,,,,,,,,,,,,,,,,,,,,,,,,,"GVM: Looks like an initial, but is listed as ""(TA, TI)""",,MAM 2,Goddard,IG1965,0214,,,aben1250,ôjemi-,AB:2,,,,*a·t-,,"relate, declare","relate, declare",initial,,,,,,,,,,,,,,,,,,,,,,,,,,,MAM 3,Goddard,IG1965,0214,,,aben1250,ôtl-,AB:2,,,,*a·nt-,,"move, change, afresh, anew","move, change, afresh, anew",initial,,,,,,,,,,,,,,,,,,,,,,,,,"GVM: Looks like an initial, but is listed as ""(TA, TI)"". MAM: this is a PA initial, so I don't know why it would say that.",,MAM diff --git a/data/models/Components.js b/data/models/Components.js index 459141db..84f80266 100644 --- a/data/models/Components.js +++ b/data/models/Components.js @@ -63,6 +63,7 @@ export default class Components extends Map { stemSubcategory: `Stem subcategory`, stemUR: `Stem UR`, subcategory: `Subcategory`, + tags: `Tags`, type: `Component Type`, UR: `UR (if given; if different)`, } diff --git a/data/tags/index.js b/data/tags/index.js index 62bee29f..02a94ad3 100644 --- a/data/tags/index.js +++ b/data/tags/index.js @@ -6,7 +6,7 @@ import { parse as parseCSV } from 'csv-parse/sync' import path from 'node:path' import ProgressBar from 'progress' import { readFile } from 'node:fs/promises' -import { setTimeout } from 'node:timers/promises' +import retryRequest from '../utilities/retryRequest.js' const csvPath = path.resolve(import.meta.dirname, `tags.csv`) @@ -70,12 +70,23 @@ async function updateSpreadsheet(lang) { const headings = originalRows.shift() const { range } = data + // Replace "Definition" heading with "Tags" + const definitionIndex = headings.indexOf(`Project Definition`) + const tagsHeader = `Tags` + + if (definitionIndex !== -1) headings.splice(definitionIndex, 1, tagsHeader) + const records = originalRows.map(row => { + const record = {} + row.forEach((cell, i) => { - record[headings[i]] = cell + const heading = headings[i] + if (heading) record[heading] = cell }) + return record + }) const cols = Components.columns @@ -84,15 +95,17 @@ async function updateSpreadsheet(lang) { for (const record of records) { - const gloss = record[cols.gloss].trim() + const gloss = record[cols.gloss]?.trim() const type = record[cols.type] const tags = tagsMap.get(gloss) if (gloss && tags && type === `initial`) { - record[cols.definition] = tags.join(`, `) + record[cols.tags] = tags.join(`, `) } - updatedRows.push(Object.values(record)) + const row = Object.values(record) + + updatedRows.push(row) } @@ -129,33 +142,8 @@ async function updateAllSpreadsheets() { const progressBar = new ProgressBar(`:bar`, { total: languages.size }) for (const key of languages.keys()) { - - let waitTime = 1 - - const makeRequest = async () => { - try { - - await updateSpreadsheet(key) - - } catch (e) { - - if (e?.response?.status === 429) { - - waitTime *= 2 - console.warn(`\nHit rate limit. Retrying after ${ waitTime }ms.`) - await setTimeout(waitTime) - await makeRequest() - - } else { - throw e - } - - } - } - - await makeRequest() // Kick off request sequence + await retryRequest(updateSpreadsheet, key) progressBar.tick() - } } diff --git a/data/utilities/retryRequest.js b/data/utilities/retryRequest.js new file mode 100644 index 00000000..891fdf3e --- /dev/null +++ b/data/utilities/retryRequest.js @@ -0,0 +1,28 @@ +import { setTimeout } from 'node:timers/promises' + +export default function retryRequest(requestFunction, ...args) { + + let waitTime = 1 + + const request = async () => { + try { + + return requestFunction(...args) + + } catch (e) { + + if (e.response?.status == 429) { + waitTime *= 2 + console.warn(`\nHit rate limit. Retrying after ${ waitTime }ms.`) + await setTimeout(waitTime) + return request() + } + + throw e + + } + } + + return request() + +}