From 98ccb7a34eba2aedd743132f80e873d755bb4363 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Tue, 9 Jan 2024 18:56:03 +0545 Subject: [PATCH 1/2] Added method to dump aoi.geojson and labels .geojson --- backend/core/models.py | 10 +++++++-- backend/core/serializers.py | 4 ++-- backend/core/tasks.py | 42 ++++++++++++++++++++++++++++--------- backend/core/utils.py | 10 +++++++-- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/backend/core/models.py b/backend/core/models.py index b1d67be1..5ad0e284 100644 --- a/backend/core/models.py +++ b/backend/core/models.py @@ -41,6 +41,7 @@ class Label(models.Model): aoi = models.ForeignKey(AOI, to_field="id", on_delete=models.CASCADE) geom = geomodels.GeometryField(srid=4326) osm_id = models.BigIntegerField(null=True, blank=True) + tags = models.JSONField(null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) @@ -101,7 +102,7 @@ class Feedback(models.Model): validators=[MinValueValidator(18), MaxValueValidator(23)] ) feedback_type = models.CharField(choices=FEEDBACK_TYPE, max_length=10) - comments = models.TextField(max_length=100,null=True,blank=True) + comments = models.TextField(max_length=100, null=True, blank=True) user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE) source_imagery = models.URLField() @@ -111,6 +112,7 @@ class DownloadStatus(models.IntegerChoices): DOWNLOADED = 1 NOT_DOWNLOADED = -1 RUNNING = 0 + training = models.ForeignKey(Training, to_field="id", on_delete=models.CASCADE) geom = geomodels.PolygonField(srid=4326) label_status = models.IntegerField(default=-1, choices=DownloadStatus.choices) @@ -123,6 +125,10 @@ class DownloadStatus(models.IntegerChoices): class FeedbackLabel(models.Model): osm_id = models.BigIntegerField(null=True, blank=True) - feedback_aoi = models.ForeignKey(FeedbackAOI, to_field="id", on_delete=models.CASCADE) + feedback_aoi = models.ForeignKey( + FeedbackAOI, to_field="id", on_delete=models.CASCADE + ) + tags = models.JSONField(null=True, blank=True) + geom = geomodels.PolygonField(srid=4326) created_at = models.DateTimeField(auto_now_add=True) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 9b08ff8b..24939aa9 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -126,7 +126,7 @@ class Meta: model = Label geo_field = "geom" # auto_bbox = True - fields = ("osm_id",) + fields = ("osm_id", "tags") class FeedbackLabelFileSerializer(GeoFeatureModelSerializer): @@ -134,7 +134,7 @@ class Meta: model = FeedbackLabel geo_field = "geom" # auto_bbox = True - fields = ("osm_id",) + fields = ("osm_id", "tags") class FeedbackFileSerializer(GeoFeatureModelSerializer): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index b78d4a96..cf74195b 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -12,15 +12,13 @@ from celery import shared_task from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( + AOISerializer, + FeedbackAOISerializer, FeedbackFileSerializer, FeedbackLabelFileSerializer, LabelFileSerializer, ) -from predictor import download_imagery,get_start_end_download_coords -from core.utils import ( - bbox, - is_dir_empty, -) +from core.utils import bbox, is_dir_empty from django.conf import settings from django.contrib.gis.db.models.aggregates import Extent from django.contrib.gis.geos import GEOSGeometry @@ -28,6 +26,7 @@ from django.utils import timezone from hot_fair_utilities import preprocess, train from hot_fair_utilities.training import run_feedback +from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -56,8 +55,8 @@ def train_model( try: ## -----------IMAGE DOWNLOADER--------- os.makedirs(settings.LOG_PATH, exist_ok=True) - if training_instance.task_id is None or training_instance.task_id.strip() == '': - training_instance.task_id=train_model.request.id + if training_instance.task_id is None or training_instance.task_id.strip() == "": + training_instance.task_id = train_model.request.id training_instance.save() log_file = os.path.join( settings.LOG_PATH, f"run_{train_model.request.id}_log.txt" @@ -77,6 +76,8 @@ def train_model( if feedback: try: aois = FeedbackAOI.objects.filter(training=feedback) + aoi_serializer = FeedbackAOISerializer(aois, many=True) + except FeedbackAOI.DoesNotExist: raise ValueError( f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", @@ -85,11 +86,12 @@ def train_model( else: try: aois = AOI.objects.filter(dataset=dataset_id) + aoi_serializer = AOISerializer(aois, many=True) + except AOI.DoesNotExist: raise ValueError( f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", ) - for obj in aois: bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: @@ -223,15 +225,35 @@ def train_model( logger.info(model.inputs) logger.info(model.outputs) - + # Convert the model to tflite for android/ios. converter = tf.lite.TFLiteConverter.from_keras_model(model) tflite_model = converter.convert() # Save the model. - with open(os.path.join(output_path, "checkpoint.tflite"), 'wb') as f: + with open(os.path.join(output_path, "checkpoint.tflite"), "wb") as f: f.write(tflite_model) + # dump labels to output folder as well + with open( + os.path.join(output_path, "labels.geojson"), + "w", + encoding="utf-8", + ) as f: + f.write(json.dumps(serialized_field.data)) + + # dump used aois as featurecollection in output + aois_geojson = { + "type": "FeatureCollection", + "features": aoi_serializer.data, + } + with open( + os.path.join(output_path, "aois.geojson"), + "w", + encoding="utf-8", + ) as f: + f.write(json.dumps(aois_geojson)) + # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path) training_instance.accuracy = float(final_accuracy) diff --git a/backend/core/utils.py b/backend/core/utils.py index 9d0a9f3b..3181eec7 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -189,6 +189,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): """Multi thread process of features""" properties = feature["properties"] osm_id = properties["osm_id"] + tags = properties["tags"] geometry = feature["geometry"] if feedback: if FeedbackLabel.objects.filter( @@ -199,7 +200,12 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ).delete() label = FeedbackLabelSerializer( - data={"osm_id": int(osm_id), "geom": geometry, "feedback_aoi": aoi_id} + data={ + "osm_id": int(osm_id), + "tags": tags, + "geom": geometry, + "feedback_aoi": aoi_id, + } ) else: @@ -211,7 +217,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False): ).delete() label = LabelSerializer( - data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id} + data={"osm_id": int(osm_id), "tags": tags, "geom": geometry, "aoi": aoi_id} ) if label.is_valid(): label.save() From 841a572b7a73efa72f589b226dba7ad8ddd524e0 Mon Sep 17 00:00:00 2001 From: kshtiijrajsharma Date: Wed, 10 Jan 2024 17:37:12 +0545 Subject: [PATCH 2/2] Fix bug on aoi serializer --- backend/core/tasks.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/backend/core/tasks.py b/backend/core/tasks.py index cf74195b..a66e4748 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -10,6 +10,15 @@ import ramp.utils import tensorflow as tf from celery import shared_task +from django.conf import settings +from django.contrib.gis.db.models.aggregates import Extent +from django.contrib.gis.geos import GEOSGeometry +from django.shortcuts import get_object_or_404 +from django.utils import timezone +from hot_fair_utilities import preprocess, train +from hot_fair_utilities.training import run_feedback +from predictor import download_imagery, get_start_end_download_coords + from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training from core.serializers import ( AOISerializer, @@ -19,14 +28,6 @@ LabelFileSerializer, ) from core.utils import bbox, is_dir_empty -from django.conf import settings -from django.contrib.gis.db.models.aggregates import Extent -from django.contrib.gis.geos import GEOSGeometry -from django.shortcuts import get_object_or_404 -from django.utils import timezone -from hot_fair_utilities import preprocess, train -from hot_fair_utilities.training import run_feedback -from predictor import download_imagery, get_start_end_download_coords logger = logging.getLogger(__name__) @@ -243,16 +244,12 @@ def train_model( f.write(json.dumps(serialized_field.data)) # dump used aois as featurecollection in output - aois_geojson = { - "type": "FeatureCollection", - "features": aoi_serializer.data, - } with open( os.path.join(output_path, "aois.geojson"), "w", encoding="utf-8", ) as f: - f.write(json.dumps(aois_geojson)) + f.write(json.dumps(aoi_serializer.data)) # now remove the ramp-data all our outputs are copied to our training workspace shutil.rmtree(base_path)