Skip to content

Commit

Permalink
Mixmix/config encoding (#165)
Browse files Browse the repository at this point in the history 
* write tests for migrating config encoding

* complete migration 02 test

* add better config encode/decode

* fix migration
  • Loading branch information
@mixmix
mixmix authored Jul 16, 2024
1 parent 3c4ab4b commit 740c3ac
Show file tree
Hide file tree
Showing 7 changed files with 198 additions and 9 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"commander": "^12.0.0",
"env-paths": "^3.0.0",
"inquirer": "8.0.0",
"is-base64": "^1.1.0",
"lodash.clonedeep": "^4.5.0",
"mkdirp": "^3.0.1",
"typescript": "^4.8.4",
Expand Down
2 changes: 1 addition & 1 deletion src/common/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export function stripHexPrefix (str: string): string {
}

export function replacer (key, value) {
if(value instanceof Uint8Array ){
if (value instanceof Uint8Array) {
return Buffer.from(value).toString('base64')
}
else return value
Expand Down
28 changes: 24 additions & 4 deletions src/config/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'
import { mkdirp } from 'mkdirp'
import { join, dirname } from 'path'
import envPaths from 'env-paths'

import isBase64 from 'is-base64'

import allMigrations from './migrations'
import { replacer } from 'src/common/utils'
Expand Down Expand Up @@ -58,15 +58,35 @@ function noop () {}

export async function get (configPath = CONFIG_PATH) {
const configBuffer = await readFile(configPath)
return JSON.parse(configBuffer.toString())
return deserialize(configBuffer.toString())
}

export function getSync (configPath = CONFIG_PATH) {
const configBuffer = readFileSync(configPath, 'utf8')
return JSON.parse(configBuffer)
return deserialize(configBuffer)
}

export async function set (config = {}, configPath = CONFIG_PATH) {
await mkdirp(dirname(configPath))
await writeFile(configPath, JSON.stringify(config, replacer))
await writeFile(configPath, serialize(config))
}

function serialize (config) {
return JSON.stringify(config, replacer, 2)
}

function deserialize (config) {
function reviver (key, value) {
if (
isBase64(value, { allowEmpty: false }) &&
value.length >= 32
// NOTE: we have to check length so we don't accidentally transform
// user simple string that are valid base64 like "registration"
) {
return Uint8Array.from(Buffer.from(value, 'base64'))
}
else return value
}

return JSON.parse(config, reviver)
}
43 changes: 43 additions & 0 deletions src/config/migrations/02.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
export const version = 2

const targetKeys = new Set(['secretKey', 'publicKey', 'addressRaw'])

export function migrate (data = {}) {
if (!isObject(data)) return data
if (isUI8A(data)) return data

const initial = isArray(data) ? [] : {}

return Object.entries(data).reduce((acc, [key, value]) => {
if (targetKeys.has(key) && !isUI8A(value)) {
acc[key] = objToUI8A(value)
}
else {
acc[key] = migrate(value)
}

return acc
}, initial)
}


function isObject (thing) {
return typeof thing === 'object'
}

function isArray (thing) {
return Array.isArray(thing)
}

function isUI8A (thing) {
return thing instanceof Uint8Array
}


function objToUI8A (obj) {
const bytes = Object.keys(obj)
.sort((a, b) => Number(a) > Number(b) ? 1 : -1)
.map(arrayIndex => obj[arrayIndex])

return new Uint8Array(bytes)
}
2 changes: 2 additions & 0 deletions src/config/migrations/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import * as migration00 from './00'
import * as migration01 from './01'
import * as migration02 from './02'

const migrations = [
migration00,
migration01,
migration02,
]

export default migrations
126 changes: 122 additions & 4 deletions tests/config.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import test from 'tape'
import { writeFile } from 'node:fs/promises'
import migrations from '../src/config/migrations'
import { migrateData, init, get, set } from '../src/config'
import { replacer } from '../src/common/utils'

// used to ensure unique test ids
let id = Date.now()
Expand Down Expand Up @@ -65,10 +66,17 @@ test('config - migrateData', async t => {
t.end()
})

const makeKey = () => new Uint8Array(
Array(32).fill(0).map((_, i) => i * 2 + 1)
)

test('config - get', async t => {
const configPath = makeTmpPath()
const config = { boop: 'doop' }
await writeFile(configPath, JSON.stringify(config))
const config = {
boop: 'doop',
secretKey: makeKey()
}
await writeFile(configPath, JSON.stringify(config, replacer))

const result = await get(configPath)
t.deepEqual(result, config, 'get works')
Expand All @@ -83,11 +91,14 @@ test('config - get', async t => {
test('config - set', async t => {
const configPath = makeTmpPath()

const config = { dog: true }
const config = {
dog: true,
secretKey: makeKey()
}
await set(config, configPath)
const actual = await get(configPath)

t.deepEqual(actual, config, 'set works')
t.deepEqual(config, actual, 'set works')
t.end()
})

Expand Down Expand Up @@ -146,3 +157,110 @@ test('config - init (migration)', async t => {

t.end()
})


test('config/migrattions/02', t => {
const initial = JSON.parse(
'{"accounts":[{"name":"Mix","address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","data":{"debug":true,"seed":"0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840","admin":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","type":"registration","verifyingKeys":["0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"],"userContext":"ADMIN_KEY","seed":"0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840","path":"","pair":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","addressRaw":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"isLocked":false,"meta":{},"publicKey":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"type":"sr25519","secretKey":{"0":120,"1":247,"2":1,"3":38,"4":246,"5":195,"6":0,"7":49,"8":84,"9":240,"10":226,"11":144,"12":66,"13":172,"14":130,"15":168,"16":237,"17":74,"18":121,"19":243,"20":49,"21":217,"22":208,"23":70,"24":160,"25":220,"26":125,"27":114,"28":230,"29":17,"30":254,"31":71,"32":158,"33":68,"34":133,"35":24,"36":119,"37":34,"38":46,"39":154,"40":85,"41":62,"42":178,"43":69,"44":206,"45":217,"46":132,"47":184,"48":8,"49":219,"50":89,"51":165,"52":189,"53":106,"54":6,"55":51,"56":112,"57":76,"58":42,"59":157,"60":146,"61":130,"62":203,"63":241}},"used":true},"registration":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","type":"registration","verifyingKeys":["0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"],"userContext":"ADMIN_KEY","seed":"0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840","path":"","pair":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","addressRaw":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"isLocked":false,"meta":{},"publicKey":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"type":"sr25519","secretKey":{"0":120,"1":247,"2":1,"3":38,"4":246,"5":195,"6":0,"7":49,"8":84,"9":240,"10":226,"11":144,"12":66,"13":172,"14":130,"15":168,"16":237,"17":74,"18":121,"19":243,"20":49,"21":217,"22":208,"23":70,"24":160,"25":220,"26":125,"27":114,"28":230,"29":17,"30":254,"31":71,"32":158,"33":68,"34":133,"35":24,"36":119,"37":34,"38":46,"39":154,"40":85,"41":62,"42":178,"43":69,"44":206,"45":217,"46":132,"47":184,"48":8,"49":219,"50":89,"51":165,"52":189,"53":106,"54":6,"55":51,"56":112,"57":76,"58":42,"59":157,"60":146,"61":130,"62":203,"63":241}},"used":true},"deviceKey":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","type":"deviceKey","verifyingKeys":["0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"],"userContext":"CONSUMER_KEY","seed":"0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840","path":"","pair":{"address":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","addressRaw":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"isLocked":false,"meta":{},"publicKey":{"0":182,"1":241,"2":171,"3":246,"4":239,"5":100,"6":192,"7":41,"8":49,"9":32,"10":10,"11":84,"12":241,"13":225,"14":183,"15":152,"16":164,"17":182,"18":176,"19":244,"20":39,"21":237,"22":74,"23":225,"24":250,"25":244,"26":187,"27":129,"28":97,"29":222,"30":33,"31":116},"type":"sr25519","secretKey":{"0":120,"1":247,"2":1,"3":38,"4":246,"5":195,"6":0,"7":49,"8":84,"9":240,"10":226,"11":144,"12":66,"13":172,"14":130,"15":168,"16":237,"17":74,"18":121,"19":243,"20":49,"21":217,"22":208,"23":70,"24":160,"25":220,"26":125,"27":114,"28":230,"29":17,"30":254,"31":71,"32":158,"33":68,"34":133,"35":24,"36":119,"37":34,"38":46,"39":154,"40":85,"41":62,"42":178,"43":69,"44":206,"45":217,"46":132,"47":184,"48":8,"49":219,"50":89,"51":165,"52":189,"53":106,"54":6,"55":51,"56":112,"57":76,"58":42,"59":157,"60":146,"61":130,"62":203,"63":241}},"used":true}}}],"selectedAccount":"5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8","endpoints":{"dev":"ws://127.0.0.1:9944","test-net":"wss://testnet.entropy.xyz"},"migration-version":1}'
)

const migrated = migrations[2].migrate(initial)

// console.log(JSON.stringify(migrated, replacer, 2))
// => {
// "accounts": [
// {
// "name": "Mix",
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "data": {
// "debug": true,
// "seed": "0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840",
// "admin": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "type": "registration",
// "verifyingKeys": [
// "0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"
// ],
// "userContext": "ADMIN_KEY",
// "seed": "0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840",
// "path": "",
// "pair": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "addressRaw": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "isLocked": false,
// "meta": {},
// "publicKey": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "type": "sr25519",
// "secretKey": "ePcBJvbDADFU8OKQQqyCqO1KefMx2dBGoNx9cuYR/keeRIUYdyIumlU+skXO2YS4CNtZpb1qBjNwTCqdkoLL8Q=="
// },
// "used": true
// },
// "registration": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "type": "registration",
// "verifyingKeys": [
// "0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"
// ],
// "userContext": "ADMIN_KEY",
// "seed": "0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840",
// "path": "",
// "pair": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "addressRaw": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "isLocked": false,
// "meta": {},
// "publicKey": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "type": "sr25519",
// "secretKey": "ePcBJvbDADFU8OKQQqyCqO1KefMx2dBGoNx9cuYR/keeRIUYdyIumlU+skXO2YS4CNtZpb1qBjNwTCqdkoLL8Q=="
// },
// "used": true
// },
// "deviceKey": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "type": "deviceKey",
// "verifyingKeys": [
// "0x03b225d2032e1dbff26316cc8b7d695b3386400d30ce004c1b42e2c28bcd834039"
// ],
// "userContext": "CONSUMER_KEY",
// "seed": "0xc4c466182b86ff1f4a16548df79c5808ab9bcde87c22c27938ac9aabc4300840",
// "path": "",
// "pair": {
// "address": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "addressRaw": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "isLocked": false,
// "meta": {},
// "publicKey": "tvGr9u9kwCkxIApU8eG3mKS2sPQn7Urh+vS7gWHeIXQ=",
// "type": "sr25519",
// "secretKey": "ePcBJvbDADFU8OKQQqyCqO1KefMx2dBGoNx9cuYR/keeRIUYdyIumlU+skXO2YS4CNtZpb1qBjNwTCqdkoLL8Q=="
// },
// "used": true
// }
// }
// }
// ],
// "selectedAccount": "5GCaN3fcL6vAQQKamHzVSorwv2XqtM3WcxosCLd9JqGVrtS8",
// "endpoints": {
// "dev": "ws://127.0.0.1:9944",
// "test-net": "wss://testnet.entropy.xyz"
// },
// "migration-version": 1
// }

const targetKeys = ['addressRaw', 'publicKey', 'secretKey']

// @ts-ignore
migrated.accounts.forEach(account => {
return Object.keys(account.data).forEach(subAccount => {
if (typeof account.data[subAccount] !== 'object') return

t.true(
targetKeys.every(targetKey => {
return account.data[subAccount].pair[targetKey] instanceof Uint8Array
}),
`migrated: ${subAccount}`
)
})
})

t.end()
})
5 changes: 5 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2378,6 +2378,11 @@ is-arrayish@^0.3.1:
resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"
integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==

is-base64@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/is-base64/-/is-base64-1.1.0.tgz#8ce1d719895030a457c59a7dcaf39b66d99d56b4"
integrity sha512-Nlhg7Z2dVC4/PTvIFkgVVNvPHSO2eR/Yd0XzhGiXCXEvWnptXlXa/clQ8aePPiMuxEGcWfzWbGw2Fe3d+Y3v1g==

is-bigint@^1.0.1:
version "1.0.4"
resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.4.tgz#08147a1875bc2b32005d41ccd8291dffc6691df3"
Expand Down

0 comments on commit 740c3ac

Please sign in to comment.