Merge pull request #62 from Mapotempo/renew_ban

New BAN support update

docker/addok/Dockerfile

@@ -19,6 +19,7 @@ RUN apt-get -q update && \
     apt-get -qy install redis-tools jq && \
     \
     pip install gunicorn addok==${ADDOK_VERSION} addok-fr==${ADDOK_FR_VERSION} addok-france==${ADDOK_FRANCE_VERSION} addok-csv==${ADDOK_CSV_VERSION} && \
+    pip install git+https://github.com/frodrigo/addok_ban_clean@master && \
     pip install git+https://github.com/frodrigo/addok_luxemburg@v${ADDOK_LUXEMBURG_VERSION} && \
     pip install git+https://github.com/Mapotempo/addok_france_clean@v${ADDOK_FRANCE_CLEAN_VERSION} && \
     pip install git+https://github.com/frodrigo/addok_luxemburg_clean@v${ADDOK_LUXEMBURG_CLEAN_VERSION} && \
@@ -37,6 +38,7 @@ RUN apt-get -q update && \
 
 EXPOSE 7878
 
+ADD ./ban_filter.sh /usr/usr/local/bin/ban_filter.sh
 ADD ./addok.conf /etc/addok/addok.conf
 
 ENTRYPOINT ["gunicorn"]

docker/addok/addok.conf

@@ -14,6 +14,7 @@ BUCKET_MIN = 20
 
 # Pipeline stream to be used.
 PROCESSORS_PYPATHS = [  # Rename in TOKEN_PROCESSORS / STRING_PROCESSORS?
+    "addok_ban_clean.ban_clean",
     "addok.helpers.text.tokenize",
     "addok.helpers.text.normalize",
     "addok_france.glue_ordinal",

docker/addok/ban_filter.sh

@@ -0,0 +1,8 @@
+jq '.name |= sub("^lieux?[ -]?dits? "; ""; "i")' | \
+jq '.name |= sub("^hameau (des |de la |de |du |le |les |la |l'"'"'|d'"'"')?"; ""; "i")'  | \
+jq '.name |= sub("^quartier (des |de la |de |du |le |les |la |l'"'"'|d'"'"')?"; ""; "i")' | \
+jq '.name |= sub("^ferme (des |de la |de |du |le |les |la |l'"'"'|d'"'"')?"; ""; "i")' | \
+jq '.name |= sub("^domaine (des |de la |de |du |le |les |la |l'"'"'|d'"'"')?"; ""; "i")' | \
+jq '.name |= sub("^village (des |de la |de |du |le |les |la |l'"'"'|d'"'"')?"; ""; "i")' | \
+jq '.name |= sub("^chemin rural(( n°?)? ?[0-9]+)?( dit)? "; "chemin "; "i")' | \
+jq -c 'del(.housenumbers[]?.id)'

docker/builder/get_bano.sh

@@ -8,8 +8,8 @@ echo "Your variables are : PROJECT : $PROJECT, DEPARTMENT: $DEP"
 
 # Download and load BAN
 if [ "${DEP}" == full ]; then
-  BAN="http://bano.openstreetmap.fr/data/full.sjson.gz"
+  BAN="https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-france.ndjson.gz"
 else
-  BAN="http://bano.openstreetmap.fr/data/bano-${DEP}.json.gz"
+  BAN="https://adresse.data.gouv.fr/data/ban/adresses/latest/addok/adresses-addok-${DEP}.ndjson.gz"
 fi
 wget "$BAN" -O "./addresses/BAN_odbl.sjson.gz"

initialize.sh

@@ -5,7 +5,7 @@ mkdir -p addresses
 # shellcheck disable=SC1091
 source ./docker/builder/get_bano.sh
 
-docker-compose -p "${PROJECT}" run --rm --entrypoint /bin/bash addok -c "zcat ./addresses/BAN_odbl.sjson.gz | jq -c 'def mapping: {\"city\":\"municipality\",\"town\":\"municipality\",\"village\":\"municipality\",\"place\":\"locality\",\"street\":\"street\"}; . + {type: mapping[.type]}' | jq -c 'del(.housenumbers[]?.id)' | addok batch"
+docker-compose -p "${PROJECT}" run --rm --entrypoint /bin/bash addok -c "zcat ./addresses/BAN_odbl.sjson.gz | /usr/usr/local/bin/ban_filter.sh | addok batch"
 
 # Patch BAN
 docker-compose -p "${PROJECT}" run --rm --entrypoint /bin/bash addok -c "ls ./addresses/*.json | xargs cat | addok batch"