From a113ae46f144d9efcc9ab7d16bf0e01c78dc039f Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Thu, 18 Apr 2024 15:25:03 +0545 Subject: [PATCH 01/35] Sep api and worker --- backend/core/urls.py | 4 ++-- backend/core/views.py | 32 +++++++++++++++++--------------- backend/requirements.txt | 13 ++++++++----- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/backend/core/urls.py b/backend/core/urls.py index c67abd63..d740e4dd 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -5,7 +5,7 @@ # now import the views.py file into this code from .views import ( AOIViewSet, - APIStatus, + # APIStatus, ConflateGeojson, DatasetViewSet, FeedbackAOIViewset, @@ -52,7 +52,7 @@ path("training/publish//", publish_training), path("prediction/", PredictionView.as_view()), path("feedback/training/submit/", FeedbackView.as_view()), - path("status/", APIStatus.as_view()), + # path("status/", APIStatus.as_view()), path("geojson2osm/", geojson2osmconverter, name="geojson2osmconverter"), path("conflate/", ConflateGeojson, name="Conflate Geojson"), path("aoi/gpx//", GenerateGpxView.as_view()), diff --git a/backend/core/views.py b/backend/core/views.py index 03f4ffe4..9526cfbe 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -12,7 +12,7 @@ from datetime import datetime from tempfile import NamedTemporaryFile -import tensorflow as tf +# import tensorflow as tf from celery import current_app from celery.result import AsyncResult from django.conf import settings @@ -60,7 +60,8 @@ ModelSerializer, PredictionParamSerializer, ) -from .tasks import train_model +# from .tasks import train_model +from celery import Celery from .utils import get_dir_size, gpx_generator, process_rawdata, request_rawdata @@ -127,9 +128,9 @@ def create(self, validated_data): validated_data["created_by"] = user # create the model instance instance = Training.objects.create(**validated_data) - + celery = Celery() # run your function here - task = train_model.delay( + task = celery.train_model.delay( dataset_id=instance.model.dataset.id, training_id=instance.id, epochs=instance.epochs, @@ -469,8 +470,9 @@ def post(self, request, *args, **kwargs): batch_size=batch_size, source_imagery=training_instance.source_imagery, ) + celery = Celery() - task = train_model.delay( + task = celery.train_model.delay( dataset_id=instance.model.dataset.id, training_id=instance.id, epochs=instance.epochs, @@ -612,16 +614,16 @@ def publish_training(request, training_id: int): return Response("Training Published", status=status.HTTP_201_CREATED) -class APIStatus(APIView): - def get(self, request): - res = { - "tensorflow_version": tf.__version__, - "No of GPU Available": len( - tf.config.experimental.list_physical_devices("GPU") - ), - "API Status": "Healthy", # static for now should be dynamic TODO - } - return Response(res, status=status.HTTP_200_OK) +# class APIStatus(APIView): +# def get(self, request): +# res = { +# "tensorflow_version": tf.__version__, +# "No of GPU Available": len( +# tf.config.experimental.list_physical_devices("GPU") +# ), +# "API Status": "Healthy", # static for now should be dynamic TODO +# } +# return Response(res, status=status.HTTP_200_OK) class GenerateGpxView(APIView): diff --git a/backend/requirements.txt b/backend/requirements.txt index 3ac6c0cd..0cb3549c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,12 +1,15 @@ django==4.1.4 -# gdal + +gdal==3.6.2 psycopg2 + + djangorestframework==3.14.0 djangorestframework-gis==1.0 dj-database-url==1.2.0 django-leaflet==0.28.3 drf-yasg==1.21.4 -Pillow +# Pillow django-environ==0.9.0 # used for environment django-filter==22.1 django-cors-headers==3.13.0 # used for enabling cors when frontend is hosted on different server / origin @@ -19,7 +22,7 @@ validators==0.20.0 gpxpy==1.5.0 hot-fair-utilities==1.2.3 geojson2osm==0.0.1 -osmconflator -orthogonalizer +osmconflator==0.0.9 +orthogonalizer==0.0.4 fairpredictor==0.0.26 -tflite-runtime==2.14.0 \ No newline at end of file +tflite-runtime==2.14.0 From 8dd7532ab79312a6a9027f29e32365fe1dee9950 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma <36752999+kshitijrajsharma@users.noreply.github.com> Date: Thu, 23 May 2024 16:16:27 +0545 Subject: [PATCH 02/35] Update frontend_build_push.yml --- .github/workflows/frontend_build_push.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/frontend_build_push.yml b/.github/workflows/frontend_build_push.yml index 79b01c75..e74f6cdc 100644 --- a/.github/workflows/frontend_build_push.yml +++ b/.github/workflows/frontend_build_push.yml @@ -6,11 +6,6 @@ on: paths: - 'frontend/**' - '.github/workflows/frontend_build_push.yml' - pull_request: - branches: [ master ] - paths: - - 'frontend/**' - - '.github/workflows/frontend_build_push.yml' permissions: id-token: write From 68622aebfe7cf52d4c65de118665269c023461b9 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Mon, 27 May 2024 13:01:42 +0200 Subject: [PATCH 03/35] fix #246 with dynamic API base --- .../components/Layout/Feedback/FeedbackAOI.js | 4 +++- .../Layout/TrainingDS/DatasetEditor/AOI.js | 17 ++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/Layout/Feedback/FeedbackAOI.js b/frontend/src/components/Layout/Feedback/FeedbackAOI.js index 4469037f..20839512 100644 --- a/frontend/src/components/Layout/Feedback/FeedbackAOI.js +++ b/frontend/src/components/Layout/Feedback/FeedbackAOI.js @@ -282,7 +282,9 @@ const FeedbackAOI = (props) => { props.sourceImagery ? "custom:" + props.sourceImagery : "Bing" - }&disable_features=boundaries&gpx=https://fair-dev.hotosm.org/api/v1/feedback-aoi/gpx/${ + }&disable_features=boundaries&gpx=${ + process.env.REACT_APP_API_BASE + }/feedback-aoi/gpx/${ layer.id }&map=10.70/18.9226/81.6991`; console.log(url); diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js index 5f8be66d..53f28be8 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js @@ -183,7 +183,9 @@ const AOI = (props) => { props.oamImagery ? "custom:" + props.oamImagery.url : "Bing" - }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ + }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=${ + process.env.REACT_APP_API_BASE + }/aoi/gpx/${ layer.aoiId }`, "_blank", @@ -275,9 +277,9 @@ const AOI = (props) => { props.oamImagery ? "custom:" + props.oamImagery.url : "Bing" - }&disable_features=boundaries&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ - layer.aoiId - }&map=10.70/18.9226/81.6991`, + }&disable_features=boundaries&gpx=${ + process.env.REACT_APP_API_BASE + }/aoi/gpx/${layer.aoiId}&map=10.70/18.9226/81.6991`, "_blank", "noreferrer" ); @@ -286,7 +288,8 @@ const AOI = (props) => { {/* */} OSM logo @@ -365,8 +368,8 @@ const AOI = (props) => { {props.mapLayers && props.mapLayers.length === 0 && ( - No TAs yet, start creating one by clicking Draw a rectangle, 3rd down at the top - left of the image/map + No TAs yet, start creating one by clicking Draw a rectangle, 3rd + down at the top left of the image/map )} From 71d4cdbcbd7b03e2a2f99f7054e4fa4a59138642 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma <36752999+kshitijrajsharma@users.noreply.github.com> Date: Mon, 27 May 2024 17:41:30 +0545 Subject: [PATCH 04/35] Update frontend_build_push.yml --- .github/workflows/frontend_build_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_build_push.yml b/.github/workflows/frontend_build_push.yml index e74f6cdc..d54b03e5 100644 --- a/.github/workflows/frontend_build_push.yml +++ b/.github/workflows/frontend_build_push.yml @@ -21,7 +21,7 @@ jobs: strategy: matrix: - node-version: [ 16 ] + node-version: [ 16.14.2 ] steps: - uses: actions/checkout@v4 From 79d14e5a9fb0f9f7181a89af0d1125abaf63c8fe Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 11:59:22 +0545 Subject: [PATCH 05/35] Add readme for API requirements --- backend/README.md | 24 +++++++----------------- backend/api-requirements.txt | 23 +++++++++++++++++++++++ backend/requirements.txt | 28 +--------------------------- backend/sample_env | 1 + 4 files changed, 32 insertions(+), 44 deletions(-) create mode 100644 backend/api-requirements.txt diff --git a/backend/README.md b/backend/README.md index 2e88a113..392dcc44 100644 --- a/backend/README.md +++ b/backend/README.md @@ -14,33 +14,27 @@ This project was bootstrapped with [Geodjango Template](https://github.com/itsk source ./env/bin/activate ##### Setup Basemodels (Ramp Supported Currently) +- Install git lfs +```bash +sudo apt-get install git-lfs +``` + - Clone Ramp Basemodel ``` git clone https://github.com/radiantearth/model_ramp_baseline.git ``` -OR Download from google drive -``` -pip install gdown -gdown --fuzzy https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing -``` - Clone Ramp - Code Note: This clone location will be your RAMP_HOME ``` git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code ``` + - Copy Basemodel checkpoint to ramp-code ``` cp -r model_ramp_baseline/data/input/checkpoint.tf ramp-code/ramp/checkpoint.tf ``` -Our Basemodel is available for public download [here](https://drive.google.com/file/d/1wvJhkiOrSlHmmvJ0avkAdu9sslFf5_I0/view?usp=sharing) - -You can unzip and move the downloaded basemodel -``` -unzip checkpoint.tf.zip -d ramp-code/ramp -``` - - Remove basemodel repo we don't need it anymore ``` @@ -136,11 +130,7 @@ pip install -r requirements.txt You will need more env variables (Such as Ramp home, Training Home) that can be found on ```.sample_env``` #### Now change your username, password and db name in settings.py accordingly to your database - python manage.py makemigrations login - python manage.py migrate login - python manage.py makemigrations core - python manage.py migrate core - python manage.py makemigrations + python manage.py makemigrations login core python manage.py migrate python manage.py runserver ### Now server will be available in your 8000 port on web, you can check out your localhost:8000/admin for admin panel diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt new file mode 100644 index 00000000..47dda210 --- /dev/null +++ b/backend/api-requirements.txt @@ -0,0 +1,23 @@ +django==4.1.4 +# gdal==3.6.2 +# psycopg2==2.9.9 +djangorestframework==3.14.0 +djangorestframework-gis==1.0 +dj-database-url==1.2.0 +django-leaflet==0.28.3 +drf-yasg==1.21.4 +django-environ==0.9.0 # used for environment +django-filter==22.1 +django-cors-headers==3.13.0 # used for enabling cors when frontend is hosted on different server / origin +osm-login-python==0.0.2 +celery==5.2.7 +redis==4.4.0 +django_celery_results==2.4.0 +flower==1.2.0 +validators==0.20.0 +gpxpy==1.5.0 +geojson2osm==0.0.1 +osmconflator==0.0.9 +orthogonalizer==0.0.4 +fairpredictor==0.0.26 +tflite-runtime==2.14.0 \ No newline at end of file diff --git a/backend/requirements.txt b/backend/requirements.txt index 0cb3549c..97733fef 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,28 +1,2 @@ -django==4.1.4 - -gdal==3.6.2 -psycopg2 - - -djangorestframework==3.14.0 -djangorestframework-gis==1.0 -dj-database-url==1.2.0 -django-leaflet==0.28.3 -drf-yasg==1.21.4 -# Pillow -django-environ==0.9.0 # used for environment -django-filter==22.1 -django-cors-headers==3.13.0 # used for enabling cors when frontend is hosted on different server / origin -osm-login-python==0.0.2 -celery==5.2.7 -redis==4.4.0 -django_celery_results==2.4.0 -flower==1.2.0 -validators==0.20.0 -gpxpy==1.5.0 +-r api-requirements.txt hot-fair-utilities==1.2.3 -geojson2osm==0.0.1 -osmconflator==0.0.9 -orthogonalizer==0.0.4 -fairpredictor==0.0.26 -tflite-runtime==2.14.0 diff --git a/backend/sample_env b/backend/sample_env index 21e8bae9..a47d9ae5 100644 --- a/backend/sample_env +++ b/backend/sample_env @@ -1,3 +1,4 @@ +DEBUG=True SECRET_KEY=yl2w)c0boi_ma-1v5)935^2#&m*r!1s9z9^*9e5co^08_ixzo6 DATABASE_URL=postgis://admin:password@localhost:5432/ai EXPORT_TOOL_API_URL=MY_RAW_DATA_URL From afc23a766bc706349896ba8ca440c0238217fa7d Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 12:15:56 +0545 Subject: [PATCH 06/35] Enable psycopg2 --- backend/api-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/api-requirements.txt b/backend/api-requirements.txt index 47dda210..0b047bea 100644 --- a/backend/api-requirements.txt +++ b/backend/api-requirements.txt @@ -1,6 +1,6 @@ django==4.1.4 # gdal==3.6.2 -# psycopg2==2.9.9 +psycopg2 djangorestframework==3.14.0 djangorestframework-gis==1.0 dj-database-url==1.2.0 From 7e5bb7e468158829e26f05149b444008823b6c17 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 14:06:47 +0545 Subject: [PATCH 07/35] add chips length in models and training --- backend/core/models.py | 1 + backend/core/tasks.py | 58 ++++++++++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index 5ad0e284..4b054ea6 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -84,6 +84,7 @@ class Training(models.Model): finished_at = models.DateTimeField(null=True, blank=True) accuracy = models.FloatField(null=True, blank=True) epochs = models.PositiveIntegerField() + chips_length = models.PositiveIntegerField(default=0) batch_size = models.PositiveIntegerField() freeze_layers = models.BooleanField(default=False) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index a3ec613b..8fbca094 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -3,23 +3,14 @@ import os import shutil import sys +import tarfile import traceback from shutil import rmtree -import tarfile import hot_fair_utilities import ramp.utils import tensorflow as tf from celery import shared_task -from django.conf import settings -from django.contrib.gis.db.models.aggregates import Extent -from django.contrib.gis.geos import GEOSGeometry -from django.shortcuts import get_object_or_404 -from django.utils import timezone -from hot_fair_utilities import preprocess, train -from hot_fair_utilities.training import run_feedback -from predictor import download_imagery, get_start_end_download_coords - from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( AOISerializer, @@ -29,6 +20,14 @@ LabelFileSerializer, ) from core.utils import bbox, is_dir_empty +from django.conf import settings +from django.contrib.gis.db.models.aggregates import Extent +from django.contrib.gis.geos import GEOSGeometry +from django.shortcuts import get_object_or_404 +from django.utils import timezone +from hot_fair_utilities import preprocess, train +from hot_fair_utilities.training import run_feedback +from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -37,6 +36,7 @@ DEFAULT_TILE_SIZE = 256 + def xz_folder(folder_path, output_filename, remove_original=False): """ Compresses a folder and its contents into a .tar.xz file and optionally removes the original folder. @@ -47,8 +47,8 @@ def xz_folder(folder_path, output_filename, remove_original=False): - remove_original: If True, the original folder is removed after compression. """ - if not output_filename.endswith('.tar.xz'): - output_filename += '.tar.xz' + if not output_filename.endswith(".tar.xz"): + output_filename += ".tar.xz" with tarfile.open(output_filename, "w:xz") as tar: tar.add(folder_path, arcname=os.path.basename(folder_path)) @@ -57,6 +57,20 @@ def xz_folder(folder_path, output_filename, remove_original=False): shutil.rmtree(folder_path) +def get_file_count(path): + try: + return len( + [ + entry + for entry in os.listdir(path) + if os.path.isfile(os.path.join(path, entry)) + ] + ) + except Exception as e: + print(f"An error occurred: {e}") + return 0 + + @shared_task def train_model( dataset_id, @@ -189,7 +203,9 @@ def train_model( rasterize_options=["binary"], georeference_images=True, ) - + training_instance.chips_length = get_file_count( + os.path.join(preprocess_output, "chips") + ) # train train_output = f"{base_path}/train" @@ -272,9 +288,19 @@ def train_model( f.write(json.dumps(aoi_serializer.data)) # copy aois and labels to preprocess output before compressing it to tar - shutil.copyfile(os.path.join(output_path, "aois.geojson"), os.path.join(preprocess_output,'aois.geojson')) - shutil.copyfile(os.path.join(output_path, "labels.geojson"), os.path.join(preprocess_output,'labels.geojson')) - xz_folder(preprocess_output, os.path.join(output_path, "preprocessed.tar.xz"), remove_original=True) + shutil.copyfile( + os.path.join(output_path, "aois.geojson"), + os.path.join(preprocess_output, "aois.geojson"), + ) + shutil.copyfile( + os.path.join(output_path, "labels.geojson"), + os.path.join(preprocess_output, "labels.geojson"), + ) + xz_folder( + preprocess_output, + os.path.join(output_path, "preprocessed.tar.xz"), + remove_original=True, + ) # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) From d10c6470087848b36ac68341569de8ff9cf0b6d9 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 09:36:51 +0000 Subject: [PATCH 08/35] Restrict using native celery function instead import the lower level function --- backend/core/views.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index a2163ac9..d920cd40 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -60,8 +60,7 @@ ModelSerializer, PredictionParamSerializer, ) -# from .tasks import train_model -from celery import Celery +from .tasks import train_model from .utils import get_dir_size, gpx_generator, process_rawdata, request_rawdata @@ -129,10 +128,8 @@ def create(self, validated_data): # create the model instance instance = Training.objects.create(**validated_data) - celery = Celery() - # run your function here - task = celery.train_model.delay( + task = train_model.delay( dataset_id=instance.model.dataset.id, training_id=instance.id, epochs=instance.epochs, @@ -474,9 +471,7 @@ def post(self, request, *args, **kwargs): batch_size=batch_size, source_imagery=training_instance.source_imagery, ) - celery = Celery() - - task = celery.train_model.delay( + task = train_model.delay( dataset_id=instance.model.dataset.id, training_id=instance.id, epochs=instance.epochs, From e784173073361550fab0d67e0f5dcff502a01076 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 09:41:55 +0000 Subject: [PATCH 09/35] tasks - save chip length when preprocessing steps is finished --- backend/core/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 8fbca094..bfbf7c3b 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -206,6 +206,8 @@ def train_model( training_instance.chips_length = get_file_count( os.path.join(preprocess_output, "chips") ) + training_instance.save() + # train train_output = f"{base_path}/train" From b642e9ad1fdba3c9329057cd06a0794244adfe05 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Thu, 30 May 2024 12:10:57 +0200 Subject: [PATCH 10/35] showing dataset size from the chip count field in trainngs table --- .../Layout/AIModels/AIModelEditor/Trainings.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/Trainings.js b/frontend/src/components/Layout/AIModels/AIModelEditor/Trainings.js index 5b3a4cf7..e53d032d 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/Trainings.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/Trainings.js @@ -50,7 +50,7 @@ const TrainingsList = (props) => { if (res.error) setError(res.error.response.statusText); else { - // console.log("gettraining", res.data); + //console.log("gettraining", res.data); return res.data; } @@ -123,17 +123,11 @@ const TrainingsList = (props) => { }, }, { - field: "c", + field: "chips_length", headerName: "DS size", flex: 1, renderCell: (params) => { - if (params.row.status === "FINISHED") - return ( - - ); + return <>{`${params.value === 0 ? "" : params.value}`}; }, }, { From 483c18b1851673f6964d585c249193ed70f9332e Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Thu, 30 May 2024 12:23:34 +0200 Subject: [PATCH 11/35] adding hashtags when opening JOSM using env variables --- frontend/src/components/Layout/Start/Prediction/Prediction.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/components/Layout/Start/Prediction/Prediction.js b/frontend/src/components/Layout/Start/Prediction/Prediction.js index 519c594c..0716def0 100644 --- a/frontend/src/components/Layout/Start/Prediction/Prediction.js +++ b/frontend/src/components/Layout/Start/Prediction/Prediction.js @@ -383,6 +383,10 @@ const Prediction = () => { loadurl.searchParams.set("top", bounds._northEast.lat); loadurl.searchParams.set("left", bounds._southWest.lng); loadurl.searchParams.set("right", bounds._northEast.lng); + loadurl.searchParams.set( + "changeset_hashtags", + process.env.REACT_APP_HASHTAG_PREFIX + ); const loadResponse = await fetch(loadurl); if (!josmResponse.ok) { From 05fa247a576f907dd17bc91fadf44da2446a4919 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Thu, 30 May 2024 12:26:11 +0200 Subject: [PATCH 12/35] Adding new env var to samples --- frontend/.env_sample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/.env_sample b/frontend/.env_sample index 79ee0133..a7b64f82 100644 --- a/frontend/.env_sample +++ b/frontend/.env_sample @@ -2,4 +2,5 @@ REACT_APP_CONNECT_ID= REACT_APP_TM_API=https://tasking-manager-tm4-production-api.hotosm.org/api/v2/projects/PROJECT_ID/tasks/ REACT_APP_ENV=Dev REACT_APP_API_BASE=http://127.0.0.1:8000/api/v1 -REACT_APP_PREDICTOR_API_BASE=http://127.0.0.1:8001 \ No newline at end of file +REACT_APP_PREDICTOR_API_BASE=http://127.0.0.1:8001 +REACT_APP_HASHTAG_PREFIX="#fAIr;#hotosm-fAIr" \ No newline at end of file From 4b2121d033a49c16e6b509dc0dfc6ab376cde7c7 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 10:32:57 +0000 Subject: [PATCH 13/35] Fix Docker build for backend --- backend/Dockerfile | 2 ++ backend/Dockerfile_CPU | 1 + 2 files changed, 3 insertions(+) diff --git a/backend/Dockerfile b/backend/Dockerfile index 11d652bf..99137694 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -20,6 +20,8 @@ RUN pip install -r /tmp/docker-requirements.txt RUN pip install --upgrade setuptools COPY requirements.txt /tmp/requirements.txt +COPY api-requirements.txt /tmp/api-requirements.txt + RUN pip install -r /tmp/requirements.txt diff --git a/backend/Dockerfile_CPU b/backend/Dockerfile_CPU index 159cae07..b6921403 100644 --- a/backend/Dockerfile_CPU +++ b/backend/Dockerfile_CPU @@ -21,6 +21,7 @@ RUN pip install -r /tmp/docker-requirements.txt RUN pip install --upgrade setuptools COPY requirements.txt /tmp/requirements.txt +COPY api-requirements.txt /tmp/api-requirements.txt RUN pip install -r /tmp/requirements.txt COPY docker/ramp/solaris /tmp/solaris From ec9dfab500b5733f637dfffe78449143b15dd4ff Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 10:42:35 +0000 Subject: [PATCH 14/35] Only push frontend once new release is made to production , also publish release images --- .github/workflows/backend_build.yml | 13 +++-- .github/workflows/docker_publish_image.yml | 8 +-- .github/workflows/frontend_build_push.yml | 59 +++++++++++----------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 7d53f39b..7983b9a6 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -3,16 +3,15 @@ on: push: branches: - master - - fix/build paths: - - 'backend/**' - - '.github/workflows/backend_build.yml' + - "backend/**" + - ".github/workflows/backend_build.yml" pull_request: branches: - master paths: - - 'backend/**' - - '.github/workflows/backend_build.yml' + - "backend/**" + - ".github/workflows/backend_build.yml" jobs: Build_on_ubuntu: @@ -111,7 +110,7 @@ jobs: pip install numpy pip install GDAL==$(gdal-config --version) --global-option=build_ext --global-option="-I/usr/include/gdal" - - name : Check Opencv version + - name: Check Opencv version run: | pip freeze | grep opencv pip install opencv-python-headless==4.7.0.68 @@ -121,7 +120,7 @@ jobs: TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }} run: | cd backend/ - + export TESTING_TOKEN=$TESTING_TOKEN python manage.py makemigrations diff --git a/.github/workflows/docker_publish_image.yml b/.github/workflows/docker_publish_image.yml index 2bdb097a..3bf592da 100644 --- a/.github/workflows/docker_publish_image.yml +++ b/.github/workflows/docker_publish_image.yml @@ -5,9 +5,11 @@ on: branches: - master paths-ignore: - - '.github/workflows/backend_build.yml' - - '.github/workflows/frontend_build.yml' - - '.github/workflows/frontend_build_push.yml' + - ".github/workflows/backend_build.yml" + - ".github/workflows/frontend_build.yml" + - ".github/workflows/frontend_build_push.yml" + release: + types: [released] env: REGISTRY: ghcr.io diff --git a/.github/workflows/frontend_build_push.yml b/.github/workflows/frontend_build_push.yml index d54b03e5..4bd108c5 100644 --- a/.github/workflows/frontend_build_push.yml +++ b/.github/workflows/frontend_build_push.yml @@ -1,11 +1,11 @@ name: Frontend Build and upload to S3 on: - push: - branches: [ master ] + release: + types: [released] paths: - - 'frontend/**' - - '.github/workflows/frontend_build_push.yml' + - "frontend/**" + - ".github/workflows/frontend_build_push.yml" permissions: id-token: write @@ -13,7 +13,6 @@ permissions: jobs: build_and_upload: - runs-on: ubuntu-latest environment: Production env: @@ -21,30 +20,30 @@ jobs: strategy: matrix: - node-version: [ 16.14.2 ] + node-version: [16.14.2] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: cd frontend/ && npm install --legacy-peer-deps - - - name: Build frontend - run: cd frontend/ && npm run build - env: - REACT_APP_API_BASE: ${{ vars.REACT_APP_API_BASE }} - REACT_APP_PREDICTOR_API_BASE: ${{ vars.REACT_APP_PREDICTOR_API_BASE }} - REACT_APP_ENV: Dev - - - name: Authenticate to AWS - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: us-east-1 - role-to-assume: ${{ secrets.AWS_OIDC_ROLE }} - role-session-name: fAIrGithub - - - name: Upload to S3 - run: cd frontend/build && aws s3 sync . s3://${{ vars.FRONTEND_BUCKET }}/ + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + run: cd frontend/ && npm install --legacy-peer-deps + + - name: Build frontend + run: cd frontend/ && npm run build + env: + REACT_APP_API_BASE: ${{ vars.REACT_APP_API_BASE }} + REACT_APP_PREDICTOR_API_BASE: ${{ vars.REACT_APP_PREDICTOR_API_BASE }} + REACT_APP_ENV: Dev + + - name: Authenticate to AWS + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + role-to-assume: ${{ secrets.AWS_OIDC_ROLE }} + role-session-name: fAIrGithub + + - name: Upload to S3 + run: cd frontend/build && aws s3 sync . s3://${{ vars.FRONTEND_BUCKET }}/ From e013b29308f8d96f8e4f49fd0beed4b75657422f Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 16:38:32 +0545 Subject: [PATCH 15/35] feat(commitzen): adds commitizen for release strategy BREAKING CHANGE: first public release --- .cz.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .cz.toml diff --git a/.cz.toml b/.cz.toml new file mode 100644 index 00000000..63d04477 --- /dev/null +++ b/.cz.toml @@ -0,0 +1,6 @@ +[tool.commitizen] +name = "cz_conventional_commits" +tag_format = "\"v$version\"" +version_scheme = "semver2" +version = "0.1.0" +update_changelog_on_bump = true From 412bd02f76c1e30e36e2d1687958cebfeff51ec5 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 16:52:31 +0545 Subject: [PATCH 16/35] =?UTF-8?q?bump:=20version=200.1.0=20=E2=86=92=201.0?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cz.toml | 2 +- CHANGELOG.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/.cz.toml b/.cz.toml index 63d04477..c8877c72 100644 --- a/.cz.toml +++ b/.cz.toml @@ -2,5 +2,5 @@ name = "cz_conventional_commits" tag_format = "\"v$version\"" version_scheme = "semver2" -version = "0.1.0" +version = "1.0.0" update_changelog_on_bump = true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..5ee95684 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +## "v1.0.0" (2024-05-30) + +### BREAKING CHANGE + +- first public release + +### Feat + +- **commitzen**: adds commitizen for release strategy + +### Fix + +- image fix + +## v0.1.0 (2024-05-30) + +## v0.0.1 (2022-12-22) From 44e5b0f79bbbfaf8ee335ecbbe64dcf79c9e556c Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma <36752999+kshitijrajsharma@users.noreply.github.com> Date: Thu, 30 May 2024 17:03:08 +0545 Subject: [PATCH 17/35] Create Release.md --- docs/Release.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/Release.md diff --git a/docs/Release.md b/docs/Release.md new file mode 100644 index 00000000..bd080f42 --- /dev/null +++ b/docs/Release.md @@ -0,0 +1,10 @@ +We use [commitizen](https://pypi.org/project/commitizen/) to manage our release version + +- Install commitizen + ```bash + pip install commitizen + ``` +- Make sure you are following the commitizen commit using `cz commit` to create new features +- Hit `cz bump` +- Push your changes `git push` +- Push your new tags `git push --tags` From 0a821118dd6d979ffbaf534cd8351466675e5c4b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 20:30:41 +0545 Subject: [PATCH 18/35] Move import dependencies --- backend/core/tasks.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index bfbf7c3b..f8c4f87b 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -7,9 +7,7 @@ import traceback from shutil import rmtree -import hot_fair_utilities -import ramp.utils -import tensorflow as tf + from celery import shared_task from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( @@ -25,8 +23,6 @@ from django.contrib.gis.geos import GEOSGeometry from django.shortcuts import get_object_or_404 from django.utils import timezone -from hot_fair_utilities import preprocess, train -from hot_fair_utilities.training import run_feedback from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -82,6 +78,13 @@ def train_model( feedback=None, freeze_layers=False, ): + #importing them here so that it won't be necessary when sending tasks + import hot_fair_utilities + import ramp.utils + import tensorflow as tf + from hot_fair_utilities import preprocess, train + from hot_fair_utilities.training import run_feedback + training_instance = get_object_or_404(Training, id=training_id) training_instance.status = "RUNNING" training_instance.started_at = timezone.now() From 9e8c553c32cd252a0edac2abc8dd70382f608f50 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 21:29:32 +0545 Subject: [PATCH 19/35] fix(tasks): only import lib when function is executed fixes the bug about not being able to run api only using api-requirements --- backend/core/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index f8c4f87b..b037ab01 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -78,7 +78,7 @@ def train_model( feedback=None, freeze_layers=False, ): - #importing them here so that it won't be necessary when sending tasks + #importing them here so that it won't be necessary when sending tasks ( api only) import hot_fair_utilities import ramp.utils import tensorflow as tf From 7e3a7509f6062442c295ca1d4dd7d9116b1b2b21 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Thu, 30 May 2024 21:32:52 +0545 Subject: [PATCH 20/35] =?UTF-8?q?bump:=20version=201.0.0=20=E2=86=92=201.0?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .cz.toml | 2 +- backend/CHANGELOG.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 backend/CHANGELOG.md diff --git a/.cz.toml b/.cz.toml index c8877c72..d24d55a3 100644 --- a/.cz.toml +++ b/.cz.toml @@ -2,5 +2,5 @@ name = "cz_conventional_commits" tag_format = "\"v$version\"" version_scheme = "semver2" -version = "1.0.0" +version = "1.0.1" update_changelog_on_bump = true diff --git a/backend/CHANGELOG.md b/backend/CHANGELOG.md new file mode 100644 index 00000000..65f61e2a --- /dev/null +++ b/backend/CHANGELOG.md @@ -0,0 +1,23 @@ +## "v1.0.1" (2024-05-30) + +### Fix + +- **tasks**: only import lib when function is executed + +## v1.0.0 (2024-05-30) + +### BREAKING CHANGE + +- first public release + +### Feat + +- **commitzen**: adds commitizen for release strategy + +### Fix + +- image fix + +## v0.1.0 (2024-05-30) + +## v0.0.1 (2022-12-22) From f6d6696d1f70ccf89b5b88262c56d0273e0a9872 Mon Sep 17 00:00:00 2001 From: Oluwanifemi Daramola <76186151+nifedara@users.noreply.github.com> Date: Sat, 22 Jun 2024 09:43:48 +0100 Subject: [PATCH 21/35] feat(pre-commit-&-pdm): adds pre-commit hooks and pdm for dependency management --- backend/.gitignore | 162 ++++++++++++++++++++++++++++++++ backend/.pre-commit-config.yaml | 23 +++++ backend/pyproject.toml | 56 +++++++++++ 3 files changed, 241 insertions(+) create mode 100644 backend/.gitignore create mode 100644 backend/.pre-commit-config.yaml create mode 100644 backend/pyproject.toml diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 00000000..3a8816c9 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm-project.org/#use-with-ide +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml new file mode 100644 index 00000000..b8b8d8ba --- /dev/null +++ b/backend/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + + # Versioning: Commit messages & changelog + - repo: https://github.com/commitizen-tools/commitizen + rev: v3.27.0 + hooks: + - id: commitizen + stages: [commit-msg] + + # Lint / autoformat: Python code + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.4.8" + hooks: + - id: ruff + - id: ruff-format + + # Trailing whitespace + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 00000000..3c813601 --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,56 @@ +[project] +name = "fAIr" +version = "1.0.1" +description = "AI Assisted Mapping" +authors = [ + {name = "HOTOSM", email = "sysadmin@hotosm.org"}, +] +dependencies = [ + "django==4.1.4", + "psycopg2", + "djangorestframework==3.14.0", + "djangorestframework-gis==1.0", + "dj-database-url==1.2.0", + "django-leaflet==0.28.3", + "drf-yasg==1.21.4", + "django-environ==0.9.0", + "django-filter==22.1", + "django-cors-headers==3.13.0", + "osm-login-python==0.0.2", + "celery==5.2.7", + "redis==4.4.0", + "django-celery-results==2.4.0", + "flower==1.2.0", + "validators==0.20.0", + "gpxpy==1.5.0", + "geojson2osm==0.0.1", + "osmconflator==0.0.9", + "orthogonalizer==0.0.4", + "fairpredictor==0.0.26", + "tflite-runtime==2.14.0", + "hot-fair-utilities==1.2.3", +] +requires-python = ">=3.10" +readme = "README.md" +license = {text = "AGPL-3.0 license"} + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" + + +[tool.pdm] +distribution = true + +[tool.pdm.dev-dependencies] +dev = [ + "commitizen>=3.27.0", + "ruff>=0.4.9", +] + +[tool.commitizen] +name = "cz_conventional_commits" +tag_format = "\"v$version\"" +version_scheme = "semver2" +version = "1.0.1" +update_changelog_on_bump = true From 95b4d3df7b85beb8dac6c6a1bf88eae752c2c276 Mon Sep 17 00:00:00 2001 From: Oluwanifemi Daramola <76186151+nifedara@users.noreply.github.com> Date: Sat, 22 Jun 2024 10:02:56 +0100 Subject: [PATCH 22/35] docs(pdm): adds doc about how to install pdm updates ReadMe with how to install pdm --- backend/README.md | 82 +++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/backend/README.md b/backend/README.md index 392dcc44..f86dbff7 100644 --- a/backend/README.md +++ b/backend/README.md @@ -3,7 +3,7 @@ ## Backend is created with [Django](https://www.djangoproject.com/) This project was bootstrapped with [Geodjango Template](https://github.com/itskshitiz321/geodjangotemplate.git) #### For Quickly Getting Started -**Note:** Depending upon your OS and Env installation will vary, This project tightly depends on [Tensorflow](https://www.tensorflow.org/install/pip) with GPU support so accordingly build your development environment +**Note:** Depending upon your OS and Env installation will vary, This project tightly depends on [Tensorflow](https://www.tensorflow.org/install/pip) with GPU support so accordingly build your development environment ### Install Python3, pip and virtualenv first ##### Skip this, step if you already have one sudo apt-get install python3 @@ -19,13 +19,13 @@ This project was bootstrapped with [Geodjango Template](https://github.com/itsk sudo apt-get install git-lfs ``` -- Clone Ramp Basemodel +- Clone Ramp Basemodel ``` git clone https://github.com/radiantearth/model_ramp_baseline.git ``` -- Clone Ramp - Code -Note: This clone location will be your RAMP_HOME +- Clone Ramp - Code +Note: This clone location will be your RAMP_HOME ``` git clone https://github.com/kshitijrajsharma/ramp-code-fAIr.git ramp-code ``` @@ -36,19 +36,19 @@ cp -r model_ramp_baseline/data/input/checkpoint.tf ramp-code/ramp/checkpoint.tf ``` -- Remove basemodel repo we don't need it anymore +- Remove basemodel repo we don't need it anymore ``` rm -rf model_ramp_baseline ``` -- Install numpy -Numpy needs to be installed before gdal +- Install numpy +Numpy needs to be installed before gdal ``` pip install numpy==1.23.5 ``` -- Install gdal and rasetrio -Based on your env : You can either use conda / setup manually on your os -for eg on ubuntu : +- Install gdal and rasetrio +Based on your env : You can either use conda / setup manually on your os +for eg on ubuntu : ``` sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt-get update sudo apt-get install gdal-bin @@ -58,12 +58,12 @@ export C_INCLUDE_PATH=/usr/include/gdal pip install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version` ``` -- Install Ramp - Dependecies +- Install Ramp - Dependecies ``` cd ramp-code && cd colab && make install ``` -- For Conda users : You may need to install rtree, gdal , rasterio & imagecodecs separately +- For Conda users : You may need to install rtree, gdal , rasterio & imagecodecs separately ``` conda install -c conda-forge rtree @@ -82,14 +82,14 @@ conda install -c conda-forge imagecodecs pip install --upgrade setuptools ``` -- Install fAIr Utilities +- Install fAIr Utilities ``` pip install hot-fair-utilities==1.0.41 ``` -**Remember In order to run fAIr , You need to configure your PC with tensorflow - GPU Support** +**Remember In order to run fAIr , You need to configure your PC with tensorflow - GPU Support** -You can check your GPU by : +You can check your GPU by : ``` import tensorflow as tf @@ -98,82 +98,88 @@ print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices(' - Install psycopg2 -Again based on your os/env you can do manual installation -for eg : on ubuntu : +Again based on your os/env you can do manual installation +for eg : on ubuntu : ``` sudo apt-get install python3-psycopg2 ``` -- Install redis server on your pc +- Install redis server on your pc ``` sudo apt install redis ``` -- Finally installl pip dependencies +- Install pdm for dependency management ``` -pip install -r requirements.txt +pip install pdm +``` + +- Finally install project dependencies + +``` +pdm install ``` ### Make sure you have postgresql installed with postgis extension enabled -#### Configure .env: +#### Configure .env: Create .env in the root backend project , and add the credentials as provided on .env_sample , Export your secret key and database url to your env - Export your database url + Export your database url ``` export DATABASE_URL=postgis://postgres:postgres@localhost:5432/ai ``` - - You will need more env variables (Such as Ramp home, Training Home) that can be found on ```.sample_env``` + + You will need more env variables (Such as Ramp home, Training Home) that can be found on ```.sample_env``` #### Now change your username, password and db name in settings.py accordingly to your database python manage.py makemigrations login core python manage.py migrate python manage.py runserver -### Now server will be available in your 8000 port on web, you can check out your localhost:8000/admin for admin panel +### Now server will be available in your 8000 port on web, you can check out your localhost:8000/admin for admin panel To login on admin panel, create your superuser and login with your credentials restarting the server python manage.py createsuperuser -## Authentication +## Authentication fAIr uses oauth2.0 Authentication using [osm-login-python](https://github.com/kshitijrajsharma/osm-login-python) 1. Get your login Url Hit ```/api/v1/auth/login/ ``` - - URL will give you login URL which you can use to provide your osm credentials and authorize fAIr + - URL will give you login URL which you can use to provide your osm credentials and authorize fAIr - After successful login you will get access-token that you can use across all osm login required endpoints in fAIr -2. Check authentication by getting back your data +2. Check authentication by getting back your data Hit ```/api/v1/auth/me/``` - - URL requires access-token as header and in return you will see your osm username, id and image url + - URL requires access-token as header and in return you will see your osm username, id and image url -## Start celery workers +## Start celery workers -- Start celery workers +- Start celery workers ``` celery -A aiproject worker --loglevel=debug -n my_worker ``` -- Monitor using flower -if you are using redis as result backend, api supports both options django / redis +- Monitor using flower +if you are using redis as result backend, api supports both options django / redis You can start flower to start monitoring your tasks ``` -celery -A aiproject --broker=redis://127.0.0.1:6379/0 flower +celery -A aiproject --broker=redis://127.0.0.1:6379/0 flower ``` -## Run Tests +## Run Tests ``` python manage.py test ``` -# Build fAIr with Docker for Development -- Install all the required drivers for your graphics to access it from containers, and check your graphics and drivers with ```nvidia-smi``` . Up to now only nvidia is Supported -- Follow docker_sample_env to create ```.env``` file in your dir +# Build fAIr with Docker for Development +- Install all the required drivers for your graphics to access it from containers, and check your graphics and drivers with ```nvidia-smi``` . Up to now only nvidia is Supported +- Follow docker_sample_env to create ```.env``` file in your dir - Build the Image ``` From 779e9cf54e856d102e4e0e6a1550770efc46f571 Mon Sep 17 00:00:00 2001 From: natrimmer Date: Wed, 26 Jun 2024 16:25:06 -0700 Subject: [PATCH 23/35] Closes #213 --- frontend/src/components/Layout/Home/Home.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/Layout/Home/Home.js b/frontend/src/components/Layout/Home/Home.js index 4211cf6d..8dd5072c 100644 --- a/frontend/src/components/Layout/Home/Home.js +++ b/frontend/src/components/Layout/Home/Home.js @@ -26,23 +26,13 @@ const GetStarted = () => { variant="body1" style={{ color: "#3D3D3D", fontSize: "18px", marginBottom: "50px" }} > - fAIr is an open AI-assisted mapping service developed by the - Humanitarian OpenStreetMap Team (HOT) that aims to improve the - efficiency and accuracy of mapping efforts for humanitarian purposes. - The service uses AI models, specifically computer vision techniques, to - detect objects such as buildings, roads, waterways, and trees from - satellite and UAV imagery. The name fAIr is derived from the following - terms: + fAIr performs mapping in the same way as human mappers using HOT's Tasking Manager. It looks at UAV imagery and produces map data that can be added to OpenStreetMap (OSM). Tests show a 100% speedup compared to manual mapping. It uses Artificial Intelligence (AI) to accomplish this.

-
    -
  • f: for freedom and free and open-source software
    • -
  • AI: for Artificial Intelligence
    • -
  • - r: for resilience and our responsibility for our communities and the - role we play within humanitarian mapping -
    • -
+ fAIr is developed by the Humanitarian OpenStreetMap Team (HOT) and all the software is free and open source. +
+
+ Before fAIr is used, it needs to be fine-tuned by training on high quality map data for a small representative part of the geographical region where it is to be used.
Date: Wed, 17 Jul 2024 16:13:35 +0545 Subject: [PATCH 24/35] feat(aoi-upload): let user upload aoi from geojson file --- frontend/package.json | 1 + .../Layout/TrainingDS/DatasetEditor/AOI.js | 188 +++++++++++------- .../TrainingDS/DatasetEditor/DatasetEditor.js | 1 + 3 files changed, 119 insertions(+), 71 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 4604544b..8e5c18ea 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,7 @@ "@mui/material": "^5.6.1", "@mui/styles": "^5.12.0", "@mui/x-data-grid": "^5.17.12", + "@terraformer/wkt": "^2.2.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^12.1.4", "@testing-library/user-event": "^13.5.0", diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js index 53f28be8..667dcbc2 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js @@ -11,24 +11,25 @@ import { ListItemText, Pagination, Snackbar, - SvgIcon, + Tooltip, Typography, + Button, } from "@mui/material"; -import Tooltip from "@mui/material/Tooltip"; import { styled } from "@mui/material/styles"; import DeleteIcon from "@mui/icons-material/Delete"; -import MapIcon from "@mui/icons-material/Map"; +import AddIcon from "@mui/icons-material/Add"; import FolderIcon from "@mui/icons-material/Folder"; import { MapTwoTone, ZoomInMap } from "@mui/icons-material"; import usePagination from "./Pagination"; import { makeStyles, withStyles } from "@material-ui/core/styles"; import ScreenshotMonitorIcon from "@mui/icons-material/ScreenshotMonitor"; - import PlaylistRemoveIcon from "@mui/icons-material/PlaylistRemove"; import { useMutation } from "react-query"; import axios from "../../../../axios"; import AOIDetails from "./AOIDetails"; import AuthContext from "../../../../Context/AuthContext"; +import * as Terraformer from "@terraformer/wkt"; + const Demo = styled("div")(({ theme }) => ({ backgroundColor: theme.palette.background.paper, })); @@ -40,11 +41,36 @@ const ListItemWithWiderSecondaryAction = withStyles({ })(ListItem); const PER_PAGE = 5; +const DEFAULT_FILTER = { + items: [], + linkOperator: "and", + quickFilterValues: [], + quickFilterLogicOperator: "and", +}; + +const postAoi = async (polygon, dataset, accessToken) => { + console.log("Posting AOI"); + console.log(dataset); + const headers = { + "Content-Type": "application/json", + "access-token": accessToken, + }; + const data = { + geom: `SRID=4326;${polygon}`, + dataset, + }; + const response = await axios.post("/aoi/", data, { headers }); + console.log(response.data); + return response.data; +}; + const AOI = (props) => { const [dense, setDense] = useState(true); const count = Math.ceil(props.mapLayers.length / PER_PAGE); let [page, setPage] = useState(1); const [openSnack, setOpenSnack] = useState(false); + const [fileError, setFileError] = useState(null); + const [geoJsonFile, setGeoJsonFile] = useState(null); let _DATA = usePagination( props.mapLayers.filter((e) => e.type === "aoi"), PER_PAGE @@ -53,7 +79,7 @@ const AOI = (props) => { setPage(p); _DATA.jump(p); }; - // console.log("_DATA", _DATA); + useEffect(() => { return () => {}; }, [props]); @@ -70,16 +96,12 @@ const AOI = (props) => { }); if (res.error) { - // setMapError(res.error.response.statusText); console.log(res.error.response.statusText); } else { - // success full fetch - return res.data; } } catch (e) { console.log("isError", e); - } finally { } }; const { mutate: mutateFetch, data: fetchResult } = @@ -106,11 +128,74 @@ const AOI = (props) => { } } catch (e) { console.log("isError", e); - } finally { } }; const { mutate: mutateDeleteAOI } = useMutation(DeleteAOI); + const handleFileUpload = async (event) => { + const file = event.target.files[0]; + if (file) { + const fileName = file.name.toLowerCase(); + if (!fileName.endsWith(".geojson")) { + setFileError("Invalid file format. Please upload a .geojson file."); + return; + } + const reader = new FileReader(); + reader.onload = async (e) => { + try { + const geoJson = JSON.parse(e.target.result); + let geometry; + + if (geoJson.type === "FeatureCollection") { + // if (geoJson.features.length > 1) { + // setFileError( + // "Feature collection contains multiple features. Only uploaded first one" + // ); + // } + // TODO : for featurecollection loop through the features and add AOI one by one + const feature = geoJson.features[0]; + if ( + feature.geometry.type !== "Polygon" && + feature.geometry.type !== "MultiPolygon" + ) { + setFileError("GeoJSON must contain a Polygon or MultiPolygon."); + return; + } + geometry = feature.geometry; + } else if (geoJson.type === "Feature") { + if ( + geoJson.geometry.type !== "Polygon" && + geoJson.geometry.type !== "MultiPolygon" + ) { + setFileError( + "Feature geometry type must be Polygon or MultiPolygon." + ); + return; + } + geometry = geoJson.geometry; + } else if ( + geoJson.type === "Polygon" || + geoJson.type === "MultiPolygon" + ) { + geometry = geoJson; + } else { + setFileError("Invalid GeoJSON format."); + return; + } + + const wkt = Terraformer.geojsonToWKT(geometry); + await postAoi(wkt, props.datasetId, accessToken); + setFileError(null); + setGeoJsonFile(null); + } catch (error) { + console.error(error); + setFileError("Error processing GeoJSON file."); + } + }; + reader.readAsText(file); + } + }; + return ( <> @@ -119,6 +204,28 @@ const AOI = (props) => { Training Areas{` (${props.mapLayers.length})`} + + + {fileError && ( + setFileError(null)}> + {fileError} + + )} {props.mapLayers && props.mapLayers.length > PER_PAGE && ( { "" )} - {/* add here a container to get the AOI status from DB */} {layer.aoiId && ( )} @@ -167,40 +273,6 @@ const AOI = (props) => { } /> - {/* - - */} - {/* - { - // mutateFetch(layer.aoiId); - // console.log("Open in Editor") - window.open( - `https://rapideditor.org/rapid#background=${ - props.oamImagery - ? "custom:" + props.oamImagery.url - : "Bing" - }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=${ - process.env.REACT_APP_API_BASE - }/aoi/gpx/${ - layer.aoiId - }`, - "_blank", - "noreferrer" - ); - }} - > - - RapiD logo - - */} { className="margin1 transparent" onClick={async (e) => { try { - // mutateFetch(layer.aoiId); - console.log("layer", layer); - const Imgurl = new URL( "http://127.0.0.1:8111/imagery" ); @@ -224,10 +293,6 @@ const AOI = (props) => { props.oamImagery.url ); const imgResponse = await fetch(Imgurl); - // bounds._southWest.lng, - // bounds._southWest.lat, - // bounds._northEast.lng, - // bounds._northEast.lat, const loadurl = new URL( "http://127.0.0.1:8111/load_and_zoom" ); @@ -270,8 +335,6 @@ const AOI = (props) => { sx={{ width: 24, height: 24 }} className="margin1 transparent" onClick={(e) => { - // mutateFetch(layer.aoiId); - // console.log("Open in Editor") window.open( `https://www.openstreetmap.org/edit/#background=${ props.oamImagery @@ -285,7 +348,6 @@ const AOI = (props) => { ); }} > - {/* */} OSM logo { className="margin1" onClick={(e) => { mutateFetch(layer.aoiId); - console.log("Call raw data API to fetch OSM data"); }} > - {/* { - - console.log("Remove labels") - }}> - - */} { return accumulator + curValue.lng; }, 0) / layer.latlngs.length; - // [lat, lng] are the centroid of the polygon props.selectAOIHandler([lat, lng], 17); }} > @@ -352,9 +403,6 @@ const AOI = (props) => { sx={{ width: 24, height: 24 }} className="margin-left-13" onClick={(e) => { - // console.log( - // `layer.aoiId ${layer.aoiId} and layer.id ${layer.id}` - // ); mutateDeleteAOI(layer.aoiId, layer.id); }} > @@ -377,7 +425,6 @@ const AOI = (props) => { open={openSnack} autoHideDuration={5000} onClose={() => { - console.log("openSnack", openSnack); setOpenSnack(false); }} message={ @@ -395,7 +442,6 @@ const AOI = (props) => { } - // action={action} color="red" anchorOrigin={{ vertical: "bottom", horizontal: "right" }} /> diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js index 10bfb8f4..10560775 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js @@ -117,6 +117,7 @@ function DatasetEditor() { mapLayers={mapLayers.filter((i) => i.type === "aoi")} selectAOIHandler={selectAOIHandler} deleteAOIButton={deleteAOIButton} + datasetId={dataset.id} > From 8adf03bb0ebe016d492ba105dd8878b232b3101c Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 17 Jul 2024 16:36:31 +0545 Subject: [PATCH 25/35] refactor(aoi): clean unused code --- .../src/components/Layout/TrainingDS/DatasetEditor/AOI.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js index 667dcbc2..2dc6860b 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js @@ -41,12 +41,6 @@ const ListItemWithWiderSecondaryAction = withStyles({ })(ListItem); const PER_PAGE = 5; -const DEFAULT_FILTER = { - items: [], - linkOperator: "and", - quickFilterValues: [], - quickFilterLogicOperator: "and", -}; const postAoi = async (polygon, dataset, accessToken) => { console.log("Posting AOI"); From c7f8d2e45debbc99c3e325440e8ee05bdd9328b9 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 17 Jul 2024 16:42:41 +0545 Subject: [PATCH 26/35] Check for frontend build , Remove old version of node --- .github/workflows/frontend_build.yml | 50 +++++++++++++--------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/.github/workflows/frontend_build.yml b/.github/workflows/frontend_build.yml index 23b091c1..ab456ba2 100644 --- a/.github/workflows/frontend_build.yml +++ b/.github/workflows/frontend_build.yml @@ -2,43 +2,41 @@ name: Frontend Build on: push: - branches: [ master ] + branches: [master] paths: - - 'frontend/**' - - '.github/workflows/frontend_build.yml' + - "frontend/**" + - ".github/workflows/frontend_build.yml" pull_request: - branches: [ master ] + branches: [master] paths: - - 'frontend/**' - - '.github/workflows/frontend_build.yml' + - "frontend/**" + - ".github/workflows/frontend_build.yml" jobs: Build_On_Ubuntu: - runs-on: ubuntu-latest env: CI: false strategy: matrix: - node-version: [ 16.14.2, 16, 18, 20 ] + node-version: [18, 20] steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: | - cd frontend/ - npm install --legacy-peer-deps - - - name: Build - run: | - cd frontend/ - npm run build - + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + run: | + cd frontend/ + npm install --legacy-peer-deps + + - name: Build + run: | + cd frontend/ + npm run build From 9d65e427f840be31ddb1fb941513f5cee6e3afe7 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 17 Jul 2024 16:45:52 +0545 Subject: [PATCH 27/35] Remove node restriction --- frontend/package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 8e5c18ea..6f992877 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -2,9 +2,6 @@ "name": "fair", "version": "0.1.0", "private": true, - "engines": { - "node": "16.14.2" - }, "dependencies": { "@emotion/react": "^11.9.0", "@emotion/styled": "^11.8.1", @@ -64,4 +61,4 @@ "last 1 safari version" ] } -} +} \ No newline at end of file From ec9752121c1088667d409dba3e693be1ff69de82 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 17 Jul 2024 16:54:40 +0545 Subject: [PATCH 28/35] Add dev dependecies ajv --- frontend/package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 6f992877..bcb8a0b7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -60,5 +60,8 @@ "last 1 firefox version", "last 1 safari version" ] + }, + "devDependencies": { + "ajv": "^7.2.4" } -} \ No newline at end of file +} From aff00d192f601bba40b97f5fd314eae623e9c1c5 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 17 Jul 2024 16:58:55 +0545 Subject: [PATCH 29/35] Restore check for 16 version of node ! --- .github/workflows/frontend_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend_build.yml b/.github/workflows/frontend_build.yml index ab456ba2..c9b1ec9a 100644 --- a/.github/workflows/frontend_build.yml +++ b/.github/workflows/frontend_build.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: - node-version: [18, 20] + node-version: [16, 18, 20] steps: - name: Checkout repository From 6ac1607de2e1dc59bd0120c668f0fbe63f720e42 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 24 Jul 2024 22:08:59 +0545 Subject: [PATCH 30/35] Upgrade treeview --- frontend/package.json | 1 + .../components/Layout/AIModels/AIModelEditor/FilesTree.js | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index bcb8a0b7..b0ab2404 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -15,6 +15,7 @@ "@mui/material": "^5.6.1", "@mui/styles": "^5.12.0", "@mui/x-data-grid": "^5.17.12", + "@mui/x-tree-view": "^7.11.0", "@terraformer/wkt": "^2.2.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^12.1.4", diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/FilesTree.js b/frontend/src/components/Layout/AIModels/AIModelEditor/FilesTree.js index 48729c50..9ff8f7cf 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/FilesTree.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/FilesTree.js @@ -1,5 +1,5 @@ import React, { useEffect, useState } from "react"; -import TreeView from "@mui/lab/TreeView"; +import { SimpleTreeView } from "@mui/x-tree-view"; import TreeItem from "@mui/lab/TreeItem"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import ExpandLessIcon from "@mui/icons-material/ExpandLess"; @@ -73,7 +73,7 @@ const FilesTree = (props) => { )} - } defaultExpandIcon={} @@ -126,7 +126,7 @@ const FilesTree = (props) => { {/* */} - + ); }; From 26cd894fbc03a8c8d242cebeb10e64291b4b0423 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 31 Jul 2024 07:35:45 +0545 Subject: [PATCH 31/35] Adds testing to backend build ci --- .github/workflows/backend_build.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 7983b9a6..db9c5137 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -1,8 +1,9 @@ -name: Backend Build +name: Backend Build and Tests on: push: branches: - master + - fix/ci-tests paths: - "backend/**" - ".github/workflows/backend_build.yml" @@ -115,17 +116,26 @@ jobs: pip freeze | grep opencv pip install opencv-python-headless==4.7.0.68 - - name: Run tests + - name: Run migrations env: TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }} run: | cd backend/ - - - export TESTING_TOKEN=$TESTING_TOKEN python manage.py makemigrations python manage.py makemigrations core python manage.py makemigrations login python manage.py migrate python manage.py migrate login python manage.py migrate core + + - name : Run tests + env : + TESTING_TOKEN: ${{ secrets.TESTING_TOKEN }} + OSM_CLIENT_ID: ${{ secrets.OSM_CLIENT_ID }} + OSM_CLIENT_SECRET: ${{ secrets.OSM_CLIENT_SECRET }} + OSM_SECRET_KEY: "" + + run : | + cd backend/ + coverage run manage.py test tests + coverage report From 1c81ec72f5e707234ad45c7f8099eb1955fc8941 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 31 Jul 2024 07:47:11 +0545 Subject: [PATCH 32/35] [ci] Fix installation requirements for teset --- .github/workflows/backend_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index db9c5137..1a5967d8 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -87,6 +87,8 @@ jobs: run: | cd backend/ pip install -r requirements.txt + pip install coverage + pip install factory-boy - name: Creating env run: | From c7408470ac134c7737b38e350ca8ed4b34b7ba61 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 31 Jul 2024 08:28:05 +0545 Subject: [PATCH 33/35] [refactor] Remove failing test cases to check if only creation works --- backend/tests/test_endpoints.py | 59 +++------------------------------ 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/backend/tests/test_endpoints.py b/backend/tests/test_endpoints.py index a0447590..4610b38d 100644 --- a/backend/tests/test_endpoints.py +++ b/backend/tests/test_endpoints.py @@ -89,58 +89,9 @@ def test_create_dataset(self): # download labels from osm for 1 - res = self.client.post( - f"{API_BASE}/label/osm/fetch/1/", "", headers=headersList - ) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - - # download labels from osm for 2 - - res = self.client.post( - f"{API_BASE}/label/osm/fetch/2/", "", headers=headersList - ) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - - # build the dataset - - build_dt_payload = {"dataset_id": 1, "zoom_level": ["19"]} - res = self.client.post( - f"{API_BASE}/dataset/image/build/", - json.dumps(build_dt_payload), - headers=json_type_header, - ) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - - # build dataset on multiple zoom levels + ## Fetch AOI - build_dt_payload = {"dataset_id": 1, "zoom_level": ["19", "20"]} - res = self.client.post( - f"{API_BASE}/dataset/image/build/", - json.dumps(build_dt_payload), - headers=json_type_header, - ) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - - # create model - - model_payload = {"name": "My test model", "dataset": 1} - res = self.client.post( - f"{API_BASE}/model/", json.dumps(model_payload), headers=json_type_header - ) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - - # create training inside model - training_payload = { - "description": "My very first training", - "epochs": 1, - "batch_size": 1, - "model": 1, - } - res = self.client.post( - f"{API_BASE}/training/", - json.dumps(training_payload), - headers=json_type_header, - ) - print(res.json()) - self.assertEqual(res.status_code, status.HTTP_201_CREATED) - # test + aoi_res = self.client.get(f"{API_BASE}/aoi/?dataset=1") + self.assertEqual(aoi_res.status_code, 200) + aoi_res_json = aoi_res.json() + self.assertEqual(len(aoi_res_json["features"]), 2) From 0c9128b9bd7f85eaa16a5dc7500b3c94125a7e08 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 31 Jul 2024 19:54:07 +0545 Subject: [PATCH 34/35] refactor(backend_build): removes the run from test branch --- backend/.DS_Store | Bin 0 -> 8196 bytes backend/core/.DS_Store | Bin 0 -> 6148 bytes backend/login/.DS_Store | Bin 0 -> 6148 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 backend/.DS_Store create mode 100644 backend/core/.DS_Store create mode 100644 backend/login/.DS_Store diff --git a/backend/.DS_Store b/backend/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..7b316faa4cfd5335544b8f2d5b8f67383bc22082 GIT binary patch literal 8196 zcmeHMJ#W-N5S_g<CUAu^; zp)k(1RyzL9Rak>~qMRPln1*yXoz%SbhAN;6r~;~hDxeDd3ku+!&864!-gl_Bs(>nR zCKcfQAw^}Id0RSbw+=M+2mqVlwk^2EIl#uG-e%sG&e~A)=}r&EhZ=vxFfJT;%I>h4 zx23Z#oQw-6=p%}}KxYW9n$(+?z6;K803h>$e0uAUP6||7&_pRMwJ}X>6|9P|5 zEbsUCr};imUVhK6U;cQb&2w}@&*Fh(K0%^#dc;`2C=Igq*bNn#QI*G)GT zRcypv!zHiZHM@B3IX#uE=GK*8VD(7IS-$k+U!hX*7?phHRflAuVBp=ZZZ}c7s z`DpnJ6)O2+RPqtMX1;Td4|9JWa^Btx(TI0B&qaKsZi7d_@7_wQSaUfY&>>BDcH?RT zBR#<11DQ|qZSOtuImSE6@2ww)VG^0XhOTXXTlUS77Qkg2Y zMY#%>yk?D;JjRWAS+{US(CW_+jkuTdMZ`z!4IYx;|F-FYd13d=InSMN#&$lxJ3L>H+d|_F%r~;=_pzFNX`TW0wxvk#;P9>i@s{;Rp0w(G2_P6mU z{%l=)DE&M3W_i8}Pqocd zfZen!wauKq&RP{v1y(2^`-6cC1|AcKcI!Z6uK>U@x~<_^?h=w?JO&;UhZupeO9i@A z_E!vc>F75;F7TK*bm_$Y^1=Sg>~AP+zmEFGh7$)4wN(XFfvf^0_ga_p|ET!>pH0#? zRX`Q^R|;@BY=(!Jl0RE>lasU7z)#>}5|=o%DQNt0Y#-z(-h*4in2`ry;4yKC9+>_J MSQ)fY1%6e5FIP5$)&Kwi literal 0 HcmV?d00001 diff --git a/backend/login/.DS_Store b/backend/login/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ae16fafc9c71591713f722c7a42d731b4740447e GIT binary patch literal 6148 zcmeHKOHRW;4E2;Q0$tD*f(25p5Qzf}RX9Np5NLv0q=ZC#7VPG3uwucIJHR0j&*M?s zC`}O?R3TgPyotx2IB%qJOhjh#SR_PaB8s4lgMADegx6U+G6)u4G=7hqt|+GkT~pTa zR>Lkbz-KqYX=T*V6V7gRepxwBrg>d8lJ-f_?^)GpIxFiLJfdfllt-U0=i9uCUwAi* z^(-8=r30$yhURpKY#%VI>6Y%>*`B`j`K{;rdUp8!_2F~fxMx+bGX2T7ou}nIRg?i` zKpEIo22itEqLHAE%78MU3@jPo`yqxh+JH{bUmX~H3jiF#>;-e~B{(JyXahPytU!!} z0wvUh7Q;w5>~Zk30iB?PlL^g-32!!`LoxpC*gy8}WUQc$%78M^W?)Yq$6Ws}`uqQO zkUl8`%D}&3zzoxAI>9G}+S>UzuC+1r63W7UPH-85i7&b OkASB^2W8+-8Tbac>RN&T literal 0 HcmV?d00001 From f3f58869824f2afa2a5a7775aef587d8a20691c0 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Wed, 31 Jul 2024 19:56:32 +0545 Subject: [PATCH 35/35] (CI): refactor the test branch --- .github/workflows/backend_build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 1a5967d8..e0749daa 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -3,7 +3,6 @@ on: push: branches: - master - - fix/ci-tests paths: - "backend/**" - ".github/workflows/backend_build.yml"