Merge pull request #200 from hotosm/feature/tag-details

Feature : Include tags in Dumps , Dumps in output

backend/core/models.py

@@ -41,6 +41,7 @@ class Label(models.Model):
     aoi = models.ForeignKey(AOI, to_field="id", on_delete=models.CASCADE)
     geom = geomodels.GeometryField(srid=4326)
     osm_id = models.BigIntegerField(null=True, blank=True)
+    tags = models.JSONField(null=True, blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
 
 
@@ -101,7 +102,7 @@ class Feedback(models.Model):
         validators=[MinValueValidator(18), MaxValueValidator(23)]
     )
     feedback_type = models.CharField(choices=FEEDBACK_TYPE, max_length=10)
-    comments = models.TextField(max_length=100,null=True,blank=True)
+    comments = models.TextField(max_length=100, null=True, blank=True)
     user = models.ForeignKey(OsmUser, to_field="osm_id", on_delete=models.CASCADE)
     source_imagery = models.URLField()
 
@@ -111,6 +112,7 @@ class DownloadStatus(models.IntegerChoices):
         DOWNLOADED = 1
         NOT_DOWNLOADED = -1
         RUNNING = 0
+
     training = models.ForeignKey(Training, to_field="id", on_delete=models.CASCADE)
     geom = geomodels.PolygonField(srid=4326)
     label_status = models.IntegerField(default=-1, choices=DownloadStatus.choices)
@@ -123,6 +125,10 @@ class DownloadStatus(models.IntegerChoices):
 
 class FeedbackLabel(models.Model):
     osm_id = models.BigIntegerField(null=True, blank=True)
-    feedback_aoi = models.ForeignKey(FeedbackAOI, to_field="id", on_delete=models.CASCADE)
+    feedback_aoi = models.ForeignKey(
+        FeedbackAOI, to_field="id", on_delete=models.CASCADE
+    )
+    tags = models.JSONField(null=True, blank=True)
+
     geom = geomodels.PolygonField(srid=4326)
     created_at = models.DateTimeField(auto_now_add=True)

backend/core/serializers.py

@@ -126,15 +126,15 @@ class Meta:
         model = Label
         geo_field = "geom"
         # auto_bbox = True
-        fields = ("osm_id",)
+        fields = ("osm_id", "tags")
 
 
 class FeedbackLabelFileSerializer(GeoFeatureModelSerializer):
     class Meta:
         model = FeedbackLabel
         geo_field = "geom"
         # auto_bbox = True
-        fields = ("osm_id",)
+        fields = ("osm_id", "tags")
 
 
 class FeedbackFileSerializer(GeoFeatureModelSerializer):

backend/core/tasks.py

@@ -10,24 +10,24 @@
 import ramp.utils
 import tensorflow as tf
 from celery import shared_task
-from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
-from core.serializers import (
-    FeedbackFileSerializer,
-    FeedbackLabelFileSerializer,
-    LabelFileSerializer,
-)
-from predictor import download_imagery,get_start_end_download_coords
-from core.utils import (
-    bbox,
-    is_dir_empty,
-)
 from django.conf import settings
 from django.contrib.gis.db.models.aggregates import Extent
 from django.contrib.gis.geos import GEOSGeometry
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
 from hot_fair_utilities import preprocess, train
 from hot_fair_utilities.training import run_feedback
+from predictor import download_imagery, get_start_end_download_coords
+
+from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training
+from core.serializers import (
+    AOISerializer,
+    FeedbackAOISerializer,
+    FeedbackFileSerializer,
+    FeedbackLabelFileSerializer,
+    LabelFileSerializer,
+)
+from core.utils import bbox, is_dir_empty
 
 logger = logging.getLogger(__name__)
 
@@ -56,8 +56,8 @@ def train_model(
     try:
         ## -----------IMAGE DOWNLOADER---------
         os.makedirs(settings.LOG_PATH, exist_ok=True)
-        if training_instance.task_id is None or training_instance.task_id.strip() == '': 
-            training_instance.task_id=train_model.request.id
+        if training_instance.task_id is None or training_instance.task_id.strip() == "":
+            training_instance.task_id = train_model.request.id
             training_instance.save()
         log_file = os.path.join(
             settings.LOG_PATH, f"run_{train_model.request.id}_log.txt"
@@ -77,6 +77,8 @@ def train_model(
             if feedback:
                 try:
                     aois = FeedbackAOI.objects.filter(training=feedback)
+                    aoi_serializer = FeedbackAOISerializer(aois, many=True)
+
                 except FeedbackAOI.DoesNotExist:
                     raise ValueError(
                         f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first",
@@ -85,11 +87,12 @@ def train_model(
             else:
                 try:
                     aois = AOI.objects.filter(dataset=dataset_id)
+                    aoi_serializer = AOISerializer(aois, many=True)
+
                 except AOI.DoesNotExist:
                     raise ValueError(
                         f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first",
                     )
-
             for obj in aois:
                 bbox_coords = bbox(obj.geom.coords[0])
                 for z in zoom_level:
@@ -223,15 +226,31 @@ def train_model(
 
             logger.info(model.inputs)
             logger.info(model.outputs)
-            
+
             # Convert the model to tflite for android/ios.
             converter = tf.lite.TFLiteConverter.from_keras_model(model)
             tflite_model = converter.convert()
 
             # Save the model.
-            with open(os.path.join(output_path, "checkpoint.tflite"), 'wb') as f:
+            with open(os.path.join(output_path, "checkpoint.tflite"), "wb") as f:
                 f.write(tflite_model)
 
+            # dump labels to output folder as well
+            with open(
+                os.path.join(output_path, "labels.geojson"),
+                "w",
+                encoding="utf-8",
+            ) as f:
+                f.write(json.dumps(serialized_field.data))
+
+            # dump used aois as featurecollection in output
+            with open(
+                os.path.join(output_path, "aois.geojson"),
+                "w",
+                encoding="utf-8",
+            ) as f:
+                f.write(json.dumps(aoi_serializer.data))
+
             # now remove the ramp-data all our outputs are copied to our training workspace
             shutil.rmtree(base_path)
             training_instance.accuracy = float(final_accuracy)

backend/core/utils.py

@@ -189,6 +189,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
     """Multi thread process of features"""
     properties = feature["properties"]
     osm_id = properties["osm_id"]
+    tags = properties["tags"]
     geometry = feature["geometry"]
     if feedback:
         if FeedbackLabel.objects.filter(
@@ -199,7 +200,12 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
             ).delete()
 
         label = FeedbackLabelSerializer(
-            data={"osm_id": int(osm_id), "geom": geometry, "feedback_aoi": aoi_id}
+            data={
+                "osm_id": int(osm_id),
+                "tags": tags,
+                "geom": geometry,
+                "feedback_aoi": aoi_id,
+            }
         )
 
     else:
@@ -211,7 +217,7 @@ def process_feature(feature, aoi_id, foreign_key_id, feedback=False):
             ).delete()
 
         label = LabelSerializer(
-            data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id}
+            data={"osm_id": int(osm_id), "tags": tags, "geom": geometry, "aoi": aoi_id}
         )
     if label.is_valid():
         label.save()