DATA: Pull latest data from Google Sheets

dwhieb · Jun 6, 2024 · 95532f1 · 95532f1
1 parent 815a5c8
commit 95532f1
Show file tree

Hide file tree

Showing 10 changed files with 1,865 additions and 1,864 deletions.
diff --git a/data/Components.js b/data/Components.js
@@ -1,3 +1,4 @@
+import { appendFile }                from 'node:fs/promises'
 import Issues                        from './Issues.js'
 import ndjson                        from '../database/NDJSON.js'
 import Orthographies                 from './Orthographies.js'
@@ -71,11 +72,16 @@ export default class Components extends Map {
     this.language = language
   }
 
-  convert(componentsCSV, tokensCSV) {
+  async convert(componentsCSV, tokensCSV) {
 
     const componentRecords = parseCSV(componentsCSV, Components.csvOptions)
     const tokenRecords     = parseCSV(tokensCSV, Components.csvOptions)
 
+    if (componentRecords.length === tokenRecords.length) {
+      const susPath = path.resolve(import.meta.dirname, `./sus.txt`)
+      await appendFile(susPath, `${ this.language }\n`, `utf8`)
+    }
+
     const records = new Map
     const cols    = Components.columns
 

diff --git a/data/DataManager.js b/data/DataManager.js
@@ -51,7 +51,7 @@ export default class DataManager {
     const componentsCSV  = await readFile(componentsPath, `utf8`)
     const tokensCSV      = await readFile(tokensPath, `utf8`)
 
-    components.convert(componentsCSV, tokensCSV)
+    await components.convert(componentsCSV, tokensCSV)
 
     await components.save()
 
@@ -93,7 +93,7 @@ export default class DataManager {
    */
   async fetchAllComponents() {
 
-    if (!this.languages) await this.loadLanguages()
+    await this.languages.load()
 
     const progressBar = new ProgressBar(`:bar`, {
       total: this.languages.size,

diff --git a/data/csv/Cree_Innu/components.csv b/data/csv/Cree_Innu/components.csv
diff --git a/data/csv/Cree_Innu/tokens.csv b/data/csv/Cree_Innu/tokens.csv
diff --git a/data/csv/Cree_Moose_Swampy/components.csv b/data/csv/Cree_Moose_Swampy/components.csv
@@ -38,7 +38,6 @@ ID,Author Surname,Source Code,Page #,ISO code,Dialect,Glottocode,Form (original
 37,Ellis,CDE2000,249,"crm, csw",,"moos1236, swam1239",milo,C-M:1,,,,,,"good, fine",,initial
 38,Ellis,CDE2000,247,"crm, csw",,"moos1236, swam1239",nah,C-M:1,,,,,,NG,,initial
 39,Ellis,CDE2000,248,"crm, csw",,"moos1236, swam1239",naka,C-M:1,,,,,,NG,,initial
-40,Ellis,CDE2000,249,"crm, csw",,"moos1236, swam1239","nân'taw, nanâtaw",C-M:1,,,,,,"tend to, seek to",,initial,,Y
 41,Ellis,CDE2000,200,"crm, csw",,"moos1236, swam1239",nataw,C-M:1,,,,,,"tend to, seek to",,initial,,,,,,,,,,,,natawâpaht-,,,V,TI,NG,,,,,,,,,,,,,,,,stem source not checked - no pdf
 42,Ellis,CDE2000,248,"crm, csw",,"moos1236, swam1239",oši,C-M:1,,,,,,NG,,initial
 43,Ellis,CDE2000,247,"crm, csw",,"moos1236, swam1239",ot,C-M:1,,,,,,NG,,initial

diff --git a/data/csv/Cree_Moose_Swampy/tokens.csv b/data/csv/Cree_Moose_Swampy/tokens.csv
@@ -38,7 +38,7 @@ Component ID,Author Surname,Source Code,Page #,ISO code,Dialect,Glottocode,Form
 37,Ellis,CDE2000,249,"crm, csw",,"moos1236, swam1239",milo,C-M:1,,,,,,"good, fine",,initial
 38,Ellis,CDE2000,247,"crm, csw",,"moos1236, swam1239",nah,C-M:1,,,,,,NG,,initial
 39,Ellis,CDE2000,248,"crm, csw",,"moos1236, swam1239",naka,C-M:1,,,,,,NG,,initial
-40,Ellis,CDE2000,249,"crm, csw",,"moos1236, swam1239","nân'taw, nanâtaw",C-M:1,,,,,,"tend to, seek to",,initial,,Y
+41,Ellis,CDE2000,249,"crm, csw",,"moos1236, swam1239","nân'taw, nanâtaw",C-M:1,,,,,,"tend to, seek to",,initial,,Y
 41,Ellis,CDE2000,200,"crm, csw",,"moos1236, swam1239",nataw,C-M:1,,,,,,"tend to, seek to",,initial,,,,,,,,,,,,natawâpaht-,,,V,TI,NG,,,,,,,,,,,,,,,,stem source not checked - no pdf
 42,Ellis,CDE2000,248,"crm, csw",,"moos1236, swam1239",oši,C-M:1,,,,,,NG,,initial
 43,Ellis,CDE2000,247,"crm, csw",,"moos1236, swam1239",ot,C-M:1,,,,,,NG,,initial

diff --git a/data/csv/Cree_Plains/tokens.csv b/data/csv/Cree_Plains/tokens.csv
diff --git a/data/csv/Meskwaki/components.csv b/data/csv/Meskwaki/components.csv
diff --git a/data/csv/Meskwaki/tokens.csv b/data/csv/Meskwaki/tokens.csv
diff --git a/data/csv/Proto_Algonquian/tokens.csv b/data/csv/Proto_Algonquian/tokens.csv