forked from nprapps/heat-income
-
Notifications
You must be signed in to change notification settings - Fork 0
/
parallel_process.sh
66 lines (40 loc) · 1.61 KB
/
parallel_process.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# FORM LIST OF CITIES IN A WAY WE CAN PIPE TO PARALLEL
rm list_of_cities.txt
for ROW in $(cat data/cities.json | jq -r '.[] | @base64')
do
_jq() {
echo ${ROW} | base64 --decode | jq -r ${1}
}
#store city and state as variables, replace " " with "-"
CITY_RAW=$(_jq '.name' | tr '[:upper:]' '[:lower:]')
STATE_RAW=$(_jq '.state' | tr '[:upper:]' '[:lower:]')
#data references
CITY=${CITY_RAW// /-}
STATE=${STATE_RAW// /-}
FULL_INPUT="$CITY $STATE"
echo $FULL_INPUT >> list_of_cities.txt
done
# # RUN PARALLEL TO PERFORM GEOANALYSIS/R ANALYSIS ON EACH CITY
# cat list_of_cities.txt | parallel -u --verbose -j 4 --colsep ' ' sh single_process.sh {}
# OBTAIN R VALUE FROM EACH CITY THEN PUT IT INTO GOOD_IMAGES_W_R.JSON
cp good_images.json good_images_w_r.json
for ROW in $(cat data/cities.json | jq -r '.[] | @base64')
do
_jq() {
echo ${ROW} | base64 --decode | jq -r ${1}
}
#store city and state as variables, replace " " with "-"
CITY_RAW=$(_jq '.name' | tr '[:upper:]' '[:lower:]')
STATE_RAW=$(_jq '.state' | tr '[:upper:]' '[:lower:]')
#data references
CITY=${CITY_RAW// /-}
STATE=${STATE_RAW// /-}
KEY="$STATE-$CITY"
# add r to cities json
R_INCOME=$(cat "data/output/analysis_out/$CITY/median_hou-R.txt")
R_NONWHITE=$(cat "data/output/analysis_out/$CITY/nonwhite_pct-R.txt")
tmp=$(mktemp)
# cat good_images_w_r.json
cat good_images_w_r.json | jq "map(if .name == \"$KEY\" then . + {\"r-income\":\"$R_INCOME\", \"r-nonwhite\": \"$R_NONWHITE\"} else . end ) "> "$tmp"
mv "$tmp" good_images_w_r.json
done