forked from brave/browser-laptop
-
Notifications
You must be signed in to change notification settings - Fork 0
/
preload-httpse.js
154 lines (140 loc) · 4.96 KB
/
preload-httpse.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// 1. Download https://www.eff.org/files/https-everywhere-latest.xpi
// 2. unzip https-everywhere-latest.xpi
// 3. cp /path/to/https-everywhere/chrome/content/rulesets.json .
// 4. npm run preload-httpse
// 5. Push httpse.json to AWS
// TODO: Automate this with a git hook.
'use strict'
var fs = require('fs')
var path = require('path')
var parseString = require('xml2js').parseString
var sqlite3 = require('sqlite3')
var levelup = require('level')
// Manually exclude sites that are broken until they are fixed in the next
// HTTPS Everywhere release.
var exclusions = {
'Nike.com.xml': 'breaks nikeplus.com',
'PJ_Media.xml': 'mixed content on https://pjmedia.com/instapundit/',
'Slashdot.xml': 'redirect loop on mobile slashdot.org',
'Delta.com.xml': 'https://delta.com does not redirect to https://www.delta.com',
'Cargo.xml': 'breaks cargocollective.com',
'TMZ.com.xml': 'breaks www.tmz.com',
'BusinessInsider.xml': 'breaks http://www.businessinsider.com/silicon-valley-100-2016-6?op=0',
'Tesco.xml': 'breaks tesco.com due to CSP mismatch',
'Vodafone.ie.xml': 'breaks pagination on http://shop.vodafone.ie/shop/phonesAndPlans/phonesAndPlansHome.jsp?subPage=phones&planFilter=onAccount',
'IDownloadBlog.xml': 'breaks http://www.idownloadblog.com/',
'EBay_static.com.xml': 'breaks suggested product image previews',
'Cisco.xml': 'breaks http://www.cisco.com/c/m/en_us/training-events/events-webinars/techwise-tv/listings.html',
'GQ.xml': 'mixed content on gq.com'
}
var rulesets = JSON.parse(fs.readFileSync('rulesets.json', 'utf8'))
// Convert XML rules to JSON
for (let id in rulesets.rulesetStrings) {
let contents = rulesets.rulesetStrings[id]
parseString(contents, function (err, result) {
if (err) {
throw new Error('FATAL: error parsing XML: ' + contents)
}
// Exclude broken rules
var ruleset = result.ruleset
if (ruleset.$.f in exclusions) {
console.log('NOTE: Excluding rule', JSON.stringify(result))
ruleset.$.default_off = exclusions[ruleset.$.f]
}
rulesets.rulesetStrings[id] = result
})
}
console.log('Writing httpse.json')
fs.writeFileSync('httpse.json', JSON.stringify(rulesets), 'utf8')
// Convert httpse.json to sqlite for mobile
console.log('creating httpse.sqlite')
var db = new sqlite3.Database('httpse.sqlite', function (err) {
if (err !== null) {
throw new Error('FATAL: could not open db: ' + err)
}
db.exec(['DROP TABLE IF EXISTS rulesets',
'CREATE TABLE rulesets (id INTEGER PRIMARY KEY, contents TEXT)',
'DROP TABLE IF EXISTS targets',
'CREATE TABLE targets (host TEXT UNIQUE, ids TEXT)'].join('; '), function (err) {
if (err !== null) {
throw new Error('FATAL: could not create tables: ' + err)
}
var rulesetStatement = db.prepare('INSERT INTO rulesets (id, contents) VALUES(?, ?)')
var targetStatement = db.prepare('INSERT INTO targets (host, ids) VALUES(?, ?)')
// TODO: Speed this up
for (var id in rulesets.rulesetStrings) {
let contents = JSON.stringify(rulesets.rulesetStrings[id])
rulesetStatement.run(id, contents)
}
for (var target in rulesets.targets) {
let ids = JSON.stringify(rulesets.targets[target])
targetStatement.run(target, ids)
}
})
})
const rmDir = function (dirPath) {
try {
var files = fs.readdirSync(dirPath)
} catch (e) {
return
}
if (files.length > 0) {
for (var i = 0; i < files.length; i++) {
var filePath = path.join(dirPath, files[i])
if (fs.statSync(filePath).isFile()) {
fs.unlinkSync(filePath)
} else {
rmDir(filePath)
}
}
}
fs.rmdirSync(dirPath)
}
console.log('creating httpse.leveldb')
rmDir('./httpse.leveldb')
const httpseLevelDB = levelup('httpse.leveldb', {compression: false, errorIfExists: true})
const ruleSets = {}
for (var id in rulesets.rulesetStrings) {
ruleSets[id] = rulesets.rulesetStrings[id]
}
let batch = httpseLevelDB.batch()
for (var target in rulesets.targets) {
let targetRuleSets = []
rulesets.targets[target].forEach((id) => {
let ruleset = ruleSets[id]
if (!ruleset.ruleset.$.default_off && !ruleset.ruleset.$.platform) {
let rule = {
r: ruleset.ruleset.rule.map((rule) => {
if (rule.$.from === '^http:' && rule.$.to === 'https:') {
return { d: 1 }
} else {
return { f: rule.$.from, t: rule.$.to }
}
})
}
if (ruleset.ruleset.exclusion) {
rule.e = ruleset.ruleset.exclusion.map((exclusion) => {
return { p: exclusion.$.pattern }
})
}
targetRuleSets = targetRuleSets.concat(rule)
}
})
let reverseTarget = target.split('.').reverse().join('.')
if (targetRuleSets.length > 0) {
batch.put(reverseTarget, JSON.stringify(targetRuleSets), {sync: true})
}
}
batch.write((err) => {
if (err) {
console.error(err)
} else {
httpseLevelDB.close((err) => {
if (err) {
console.error(err)
} else {
console.log('done')
}
})
}
})