other: improve data build workflow

basically just merged package/data/ and the originally untracked tools/#data/ into data/
DGCK81LNN · Mar 23, 2024 · ee2dab9 · ee2dab9
1 parent a854666
commit ee2dab9
Show file tree

Hide file tree

Showing 20 changed files with 151,762 additions and 3 deletions.
diff --git a/data/210430.tsv b/data/210430.tsv
diff --git a/data/211205.tsv b/data/211205.tsv
diff --git a/data/220708.tsv b/data/220708.tsv
diff --git a/data/221206.tsv b/data/221206.tsv
diff --git a/data/230529.tsv b/data/230529.tsv
diff --git a/data/230823.tsv b/data/230823.tsv
diff --git a/data/230901.tsv b/data/230901.tsv
diff --git a/data/231021.tsv b/data/231021.tsv
diff --git a/data/231116.tsv b/data/231116.tsv
diff --git a/data/240204.tsv b/data/240204.tsv
diff --git a/data/240205.tsv b/data/240205.tsv
diff --git a/data/240206.tsv b/data/240206.tsv
diff --git a/data/240319.tsv b/data/240319.tsv
diff --git a/package/data/dict.tsv → data/dict.tsv b/package/data/dict.tsv → data/dict.tsv
diff --git a/package/data/misc.json → data/misc.json b/package/data/misc.json → data/misc.json
diff --git a/data/西丁.tsv b/data/西丁.tsv
diff --git a/package/README.md b/package/README.md
@@ -469,3 +469,23 @@ interface Transcriber {
 ~~~
 
 具有 `transcribe` 方法，其接受字符串作为第一个参数、返回 `TranscribeResult` 的类型。本模块中的各转写器类都实现该接口。
+
+## 维护
+
+新字表发布时：
+
+ 1. 将字表以 TSV 格式（无标题行，第一列为汉字，第二列为希顶聊天字母）保存到 <code>data/<var>6位日期</var>.tsv</code>。
+
+ 2. 执行 <code>ruby tools/sort.rb data/<var>xxxxxx</var>.tsv</code>，统一字表数据的排序（希顶拼写按希顶字母表顺序简单排序）。
+
+ 3. 在 `data` 目录中，执行 `node ../tools/append.mjs dict.tsv`：此过程可能会移除部分条目的汉希提示，程序会输出相应的提示，在下一步中可能需要将其补回到适当位置。
+
+ 4. 检查 `data/dict.tsv` 的 `git diff`，补充缺少的注释、提示（如新增多音字时，填写各读音的释义或对应普通话读音等）。
+
+      * 使用 `tools/dictgitdiff.sh data/dict.tsv` 可省略 diff 中多余的上下文。当标准输出为控制台时，该脚本会自动使用环境中的 <code>python -m [pygments](https://pygments.org/)</code> 或 [`rougify`](https://rouge.jneen.net/) 为 diff 添加颜色。
+
+ 5. 如果在注释中使用了数字表示拼音声调的写法（如 `pin1 yin1`），执行 `ruby tools/pinyin.rb data/dict.tsv data/dict.tsv`，将其转换为声调标号。
+
+ 6. 更新 README 和 `site/app.vue` 中注明的字表版本以及更新记录。
+
+ 7. 修改 `package/package.json` 中的包版本号，执行 `npm run build:package` 或其他合适的构建指令，为新的版本号创建 Git 标签，发布包更新。
diff --git a/package/build/data.js b/package/build/data.js
@@ -13,7 +13,7 @@ const startTime = Date.now()
 const outStream = fs.createWriteStream(pr("../src/data.json"))
 
 const convertPromise = new Promise(resolve => {
-  const inStream = fs.createReadStream(pr("../data/dict.tsv"))
+  const inStream = fs.createReadStream(pr("../../data/dict.tsv"))
   const rl = readline.createInterface({
     input: inStream,
     terminal: false,
@@ -51,7 +51,7 @@ const convertPromise = new Promise(resolve => {
   })
 })
 
-const miscPromise = fs.promises.readFile(pr("../data/misc.json")).then(b => {
+const miscPromise = fs.promises.readFile(pr("../../data/misc.json")).then(b => {
   return JSON.stringify(JSON.parse(b.toString()))
 })
 

diff --git a/tools/append.mjs b/tools/append.mjs
@@ -49,6 +49,11 @@ for (const [h, xs] of entries) {
       e.xh = "-"
       if (!e.n?.includes("旧拼写"))
         e.n = (e.n || "") + (e.n ? "，" : "") + "旧拼写"
+    } else if (table[h][latest]?.includes(x) && e.hh === "-" && e.xh === "-") {
+      console.warn(`Entry ${h} — ${x} is no longer deprecated, removing hints`)
+      e.hh = ""
+      e.xh = ""
+      e.n = e.n.replace(/，?旧拼写/, "")
     }
   }
   dictTable[h].sort(

diff --git a/tools/dictgitdiff.sh b/tools/dictgitdiff.sh
@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
 
+function highlight {
+  if [ -t 1 ]; then
+    if   python -c "import pygments"  &> /dev/null; then exec python -m pygments -l diff
+    elif rougify --version            &> /dev/null; then exec rougify            -l diff
+    fi
+  fi
+  exec cat
+}
+
 git diff "$@" > "$0.tmp"
-grep -P "^[ +-]($(grep -Po '(?<=^[+-])[^+-](?=\t)' "$0.tmp" | uniq | paste -sd'|'))" "$0.tmp"
+grep -P "^[ +-]($(grep -Po '(?<=^[+-])[^+-](?=\t)' "$0.tmp" | sort -u | paste -sd'|'))" "$0.tmp" | highlight
 rm "$0.tmp"