From dca22f0569b6daa389da0cd22287d0ab59c3fbcf Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Thu, 10 Aug 2023 13:04:48 +0200 Subject: [PATCH 01/17] in dataset and mapping pages, show tile separaters --- .../Layout/TrainingDS/DatasetEditor/DatasetEditor.css | 5 ----- frontend/src/index.css | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css index 31f7c992..5c0c8ca8 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css @@ -2,11 +2,6 @@ text-align: center; } -.leaflet-tile-container img -{ - padding: 1px; -} - .margin1 { margin-left: 2px !important; diff --git a/frontend/src/index.css b/frontend/src/index.css index 82e48df5..a632d284 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -146,3 +146,8 @@ code { { background-color: transparent !important; } + +.leaflet-tile-container img, .leaflet-pane.leaflet-tile-pane img +{ + padding: 1px; +} From 2c953aff8d0341eed77cd38b2eb45b6a3c7bab6d Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Thu, 10 Aug 2023 19:14:28 +0200 Subject: [PATCH 02/17] Feedback page based on training Id and initial map for feedback AOIs --- .../AIModels/AIModelEditor/AIModelEditor.js | 40 ++- .../AIModels/AIModelEditor/FeedbackPopup.js | 20 +- .../components/Layout/AIModels/AIModels.js | 77 ++-- .../components/Layout/Feedback/Feedback.js | 274 +++++++++++++++ .../components/Layout/Feedback/FeedbackAOI.js | 330 ++++++++++++++++++ .../Layout/Feedback/FeedbackAOIDetails.js | 50 +++ .../components/Layout/Feedback/Pagination.js | 29 ++ .../DatasetEditor/DatasetEditor.css | 8 - frontend/src/index.css | 14 + frontend/src/routes.js | 7 +- 10 files changed, 776 insertions(+), 73 deletions(-) create mode 100644 frontend/src/components/Layout/Feedback/Feedback.js create mode 100644 frontend/src/components/Layout/Feedback/FeedbackAOI.js create mode 100644 frontend/src/components/Layout/Feedback/FeedbackAOIDetails.js create mode 100644 frontend/src/components/Layout/Feedback/Pagination.js diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js index ed4b03e9..0aa494db 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js @@ -24,6 +24,7 @@ import DatasetCurrent from "./DatasetCurrent"; import FeedbackToast from "./FeedbackToast"; import FeedbackPopup from "./FeedbackPopup"; import FormGroup from "@mui/material/FormGroup"; +import LoadingButton from "@mui/lab/LoadingButton/LoadingButton"; const AIModelEditor = (props) => { let { id } = useParams(); @@ -50,6 +51,7 @@ const AIModelEditor = (props) => { if (res.error) setError(res.error.response.statusText); else { // console.log("getmodel", res.data); + getFeedbackCount(res.data.published_training); return res.data; } } catch (e) { @@ -78,23 +80,22 @@ const AIModelEditor = (props) => { refetchInterval: 60000, } ); - const getFeedbackCount = async () => { + const getFeedbackCount = async (trainingId) => { try { - const response = await axios.get( - `/feedback/?training=${data.published_training}` - ); + const response = await axios.get(`/feedback/?training=${trainingId}`); setFeedbackData(response.data); - const feedbackCount = response.data.features.length; - setFeedbackCount(feedbackCount); + console.log(`/feedback/?training=${trainingId}`, response.data); + setFeedbackCount(response.data.features.length); } catch (error) { console.error("Error fetching feedback information:", error); } }; - useEffect(() => { - if (data?.published_training) { - getFeedbackCount(); - } - }, [data]); + + // useEffect(() => { + // if (data?.published_training) { + // getFeedbackCount(); + // } + // }, [data]); const handleFeedbackClick = async (trainingId) => { getFeedbackCount(); @@ -153,13 +154,13 @@ const AIModelEditor = (props) => { {data && ( - {data.published_training && ( + {/* {data.published_training && ( - )} + )} */} Model ID: {data.id} @@ -316,7 +317,7 @@ const AIModelEditor = (props) => { }} /> - + {/* Freeze Layers @@ -333,7 +334,7 @@ const AIModelEditor = (props) => { /> - + */} @@ -353,18 +354,21 @@ const AIModelEditor = (props) => { - + {error && ( diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/FeedbackPopup.js b/frontend/src/components/Layout/AIModels/AIModelEditor/FeedbackPopup.js index 07a26a58..958dbc21 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/FeedbackPopup.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/FeedbackPopup.js @@ -21,41 +21,25 @@ const useStyles = makeStyles((theme) => ({ })); const FeedbackPopup = ({ - feedbackData, onClose, sourceImagery, trainingId, + feedbackData, }) => { const classes = useStyles(); const [open, setOpen] = useState(true); const [loading, setLoading] = useState(false); const [freezeLayers, setFreezeLayers] = useState(false); - const { accessToken } = useContext(AuthContext); const actionCounts = { CREATE: 0, MODIFY: 0, ACCEPT: 0, }; + const [epochs, setEpochs] = useState(2); const [batchSize, setBatchSize] = useState(1); - feedbackData.features.forEach((feature) => { - switch (feature.properties.action) { - case "CREATE": - actionCounts.CREATE++; - break; - case "MODIFY": - actionCounts.MODIFY++; - break; - case "ACCEPT": - actionCounts.ACCEPT++; - break; - default: - break; - } - }); - const handleClose = () => { setOpen(false); onClose(); diff --git a/frontend/src/components/Layout/AIModels/AIModels.js b/frontend/src/components/Layout/AIModels/AIModels.js index 318ca618..f9e4d367 100644 --- a/frontend/src/components/Layout/AIModels/AIModels.js +++ b/frontend/src/components/Layout/AIModels/AIModels.js @@ -1,32 +1,55 @@ -import { Box, Button } from '@mui/material'; -import React, { useContext } from 'react' -import { Route, Routes } from 'react-router-dom'; -import AuthContext from '../../../Context/AuthContext'; -import AIModelEditor from './AIModelEditor/AIModelEditor'; -import AIModelNew from './AIModelNew/AIModelNew'; -import AIModelsList from './AIModelsList/AIModelsList'; +import { Box, Button } from "@mui/material"; +import React, { useContext } from "react"; +import { Route, Routes } from "react-router-dom"; +import AuthContext from "../../../Context/AuthContext"; +import AIModelEditor from "./AIModelEditor/AIModelEditor"; +import AIModelNew from "./AIModelNew/AIModelNew"; +import AIModelsList from "./AIModelsList/AIModelsList"; +import Feedback from "../Feedback/Feedback"; -const AIModels = props => { +const AIModels = (props) => { + const { accessToken } = useContext(AuthContext); - const {accessToken} = useContext(AuthContext) + return ( + <> + {accessToken && ( + + {/* } /> */} + } + /> + } + /> + } /> + } + /> + + )} - return <> - -{accessToken && - {/* } /> */} - } /> - } /> - } /> - } - - {!accessToken && -
- arrow -
} - - + {!accessToken && ( +
+ arrow +
+ )} -} + ); +}; -export default AIModels; \ No newline at end of file +export default AIModels; diff --git a/frontend/src/components/Layout/Feedback/Feedback.js b/frontend/src/components/Layout/Feedback/Feedback.js new file mode 100644 index 00000000..93218d44 --- /dev/null +++ b/frontend/src/components/Layout/Feedback/Feedback.js @@ -0,0 +1,274 @@ +import React, { useContext, useEffect, useState } from "react"; +import { useParams } from "react-router-dom"; +import AuthContext from "../../../Context/AuthContext"; +import axios from "../../../axios"; +import { useMutation, useQuery } from "react-query"; +import { Alert, AlertTitle, Grid, Typography } from "@mui/material"; +import { EditControl } from "react-leaflet-draw"; +import { GeoJSON } from "react-leaflet"; +import L from "leaflet"; +import "leaflet-draw/dist/leaflet.draw.css"; +import { + FeatureGroup, + LayersControl, + MapContainer, + Polygon, + TileLayer, + useMapEvents, +} from "react-leaflet"; +import FeedbackAOI from "./FeedbackAOI"; +import { approximateGeom, converToGeoPolygon } from "../../../utils"; +const Feedback = (props) => { + let { id, trainingId } = useParams(); + + const { accessToken } = useContext(AuthContext); + + const [sourceImagery, setSourceImagery] = useState(""); + const getSourceImagery = async () => { + try { + const response = await axios.get(`/training/${trainingId}/`); + setSourceImagery(response.data.source_imagery); + getImagery(response.data.source_imagery); + } catch (error) { + console.error(error); + } + }; + const [feedbackData, setFeedbackData] = useState(null); + const { + mutate, + data: trainingData, + isLoading: isLoadingTraining, + } = useMutation(getSourceImagery); + const getFeedback = async () => { + try { + const headers = { + "access-token": accessToken, + }; + const res = await axios.get(`/feedback/?training=${trainingId}`, null, { + headers, + }); + + if (res.error) { + } else { + // console.log(`/feedback/?training=${trainingId}`, res.data); + mutate(); + setFeedbackData(res.data); + } + } catch (e) { + console.log("isError", e); + } finally { + } + }; + // const { data: feedbackData, isLoading } = useQuery( + // "getFeedback" + trainingId, + // getFeedback, + // { + // refetchInterval: 120000, + // } + // ); + useEffect(() => { + getFeedback(); + + return () => {}; + }, []); + + const [zoom, setZoom] = useState(15); + const [bounds, setBounds] = useState({}); + + const [map, setMap] = useState(null); + + const [windowSize, setWindowSize] = useState([ + window.innerWidth, + window.innerHeight, + ]); + const { mutate: getImagery, data: oamImagery } = useMutation(async (url) => { + const res = await axios.get(url.replace("/{z}/{x}/{y}", "")); + if (res.error) { + // setError(res.error.response.statusText); + return; + } + if (map) + map.setView([res.data.center[1], res.data.center[0]], res.data.center[2]); + console.log("OAM data", res.data, "map", map); + return res.data; + }); + + // useEffect(() => { + // console.log("map && oamImagery", map, oamImagery); + + // if (map && oamImagery) { + // // map.setView( + // // ([oamImagery.center[1], oamImagery.center[0]], oamImagery.center[2]) + // // ); + // // setZoom(oamImagery.center[2]); + // } + // return () => {}; + // }, [map, oamImagery]); + + function getFeatureStyle(feature) { + return { + color: "red", + weight: 3, + }; + } + const [error, setError] = useState(""); + const createDB = async ({ geom, leafletId }) => { + try { + const body = { + geom: geom, + training: trainingId, + source_imagery: sourceImagery, + }; + + const headers = { + "access-token": accessToken, + }; + const res = await axios.post(`/feedback-aoi/`, body, { headers }); + console.log("res ", res); + + if (res.error) { + setError(JSON.stringify(res.error)); + } else { + // add aoi ID to the state after insert + setRefresh(Math.random()); + return res.data; + } + } catch (e) { + console.log("isError", e); + setError(e); + } finally { + } + }; + const { mutate: mutateCreateDB, data: createResult } = useMutation(createDB); + const _onCreate = (e, str) => { + console.log("_onCreate", e); + const { layerType, layer } = e; + + const { _leaflet_id } = layer; + + // call the API and add the AOI to DB + const newAOI = { + id: _leaflet_id, + type: str, + latlngs: layer.getLatLngs()[0], + area: L.GeometryUtil.geodesicArea(layer.getLatLngs()[0]), + }; + const points = JSON.stringify( + converToGeoPolygon([newAOI])[0][0].reduce( + (p, c, i) => p + c[1] + " " + c[0] + ",", + "" + ) + ).slice(1, -2); + // console.log("points",points) + const approximated = str === "aoi" ? approximateGeom(points) : points; + const polygon = "SRID=4326;POLYGON((" + approximated + "))"; + + console.log("converToPolygon([layer])", polygon); + mutateCreateDB({ + geom: polygon, + leafletId: _leaflet_id, + polyTemp: converToGeoPolygon([newAOI])[0][0], + }); + }; + const [refresh, setRefresh] = useState(Math.random()); + return ( + <> + {!feedbackData && "Loading ..."} + + {feedbackData && ( + + + {oamImagery && ( + + {/* */} + {oamImagery && ( + + )} + + + {/* e.type === "aoi") + )} + /> */} + + + { + // _onFeatureGroupReady(reactFGref, geoJsonLoadedFile); + // if (zoom >= 19) { + // _onFeatureGroupReadyLabels(reactFGref, geoJsonLoadedLabels); + // } else { + // setgeoJsonLoadedLabels(null); + // } + }} + > + + + + )} + + + + + + Mappers feedback is show in red in the map + + + + + {oamImagery && ( + + )} + + + + )} + + ); +}; + +export default Feedback; diff --git a/frontend/src/components/Layout/Feedback/FeedbackAOI.js b/frontend/src/components/Layout/Feedback/FeedbackAOI.js new file mode 100644 index 00000000..b287a9d4 --- /dev/null +++ b/frontend/src/components/Layout/Feedback/FeedbackAOI.js @@ -0,0 +1,330 @@ +import React, { useContext, useEffect, useState } from "react"; +import { + Avatar, + Grid, + IconButton, + List, + ListItem, + ListItemAvatar, + ListItemSecondaryAction, + ListItemText, + Pagination, + SvgIcon, + Typography, +} from "@mui/material"; +import Tooltip from "@mui/material/Tooltip"; +import { styled } from "@mui/material/styles"; +import DeleteIcon from "@mui/icons-material/Delete"; +import MapIcon from "@mui/icons-material/Map"; +import FolderIcon from "@mui/icons-material/Folder"; +import { MapTwoTone, ZoomInMap } from "@mui/icons-material"; +import usePagination from "./Pagination"; +import { makeStyles, withStyles } from "@material-ui/core/styles"; +import ScreenshotMonitorIcon from "@mui/icons-material/ScreenshotMonitor"; + +import PlaylistRemoveIcon from "@mui/icons-material/PlaylistRemove"; +import { useMutation, useQuery } from "react-query"; +import axios from "../../../axios"; +import AOIDetails from "./FeedbackAOIDetails"; +import AuthContext from "../../../Context/AuthContext"; +import FeedbackAOIDetails from "./FeedbackAOIDetails"; +const Demo = styled("div")(({ theme }) => ({ + backgroundColor: theme.palette.background.paper, +})); + +const ListItemWithWiderSecondaryAction = withStyles({ + secondaryAction: { + paddingRight: 96, + }, +})(ListItem); + +const PER_PAGE = 5; +const FeedbackAOI = (props) => { + const [dense, setDense] = useState(true); + const getFeedbackAOIs = async () => { + try { + const res = await axios.get( + `/feedback-aoi/?training=${props.trainingId}` + ); + + if (res.error) { + // setError(res.error.response.statusText); + } else { + // console.log("gettraining", res.data); + + return res.data; + } + } catch (e) { + // setError(e); + } finally { + } + }; + const { data, isLoading, refetch } = useQuery( + "getFeedbackAOIs" + props.trainingId, + getFeedbackAOIs, + { refetchInterval: 60000 } + ); + const count = Math.ceil(data ? data?.features.length / PER_PAGE : 0); + let [page, setPage] = useState(1); + let _DATA = usePagination(data ? data?.features : [], PER_PAGE); + const handleChange = (e, p) => { + setPage(p); + _DATA.jump(p); + }; + // console.log("_DATA", _DATA); + useEffect(() => { + refetch(); + return () => {}; + }, [props.refresh]); + + const { accessToken } = useContext(AuthContext); + const fetchOSMLebels = async (aoiId) => { + try { + const headers = { + "access-token": accessToken, + }; + + const res = await axios.post(`/label/osm/fetch/${aoiId}/`, null, { + headers, + }); + + if (res.error) { + // setMapError(res.error.response.statusText); + console.log(res.error.response.statusText); + } else { + // success full fetch + + return res.data; + } + } catch (e) { + console.log("isError", e); + } finally { + } + }; + const { mutate: mutateFetch, data: fetchResult } = + useMutation(fetchOSMLebels); + const DeleteAOI = async (id) => { + try { + const headers = { + "access-token": accessToken, + }; + + const res = await axios.delete(`/feedback-aoi/${id}`, { + headers, + }); + + if (res.error) { + console.log(res); + console.log(res.error.response.statusText); + } else { + // success full fetch + + refetch(); + return res.data; + } + } catch (e) { + console.log("isError", e); + } finally { + } + }; + const { mutate: mutateDeleteAOI } = useMutation(DeleteAOI); + return ( + <> + + + + List of feedback area of Interests{` (${data?.features.length})`} + + + + {data && data.features && data.features.length > PER_PAGE && ( + + )} + + {data && + data.features && + data.features.length > 0 && + _DATA.currentData().map((layer) => ( + + + + + + + + Area: {parseInt(layer.area).toLocaleString()} sqm
+ + {parseInt(layer.area) < 5000 ? ( + <> + Area seems to be very small for an AOI +
+ Make sure it is not a Label + + ) : ( + "" + )} + + {/* add here a container to get the AOI status from DB */} + {layer.id && ( + + )} + + } + /> + + {/* + + */} + + { + // mutateFetch(layer.aoiId); + // console.log("Open in Editor") + window.open( + `https://rapideditor.org/rapid#background=${ + props.oamImagery + ? "custom:" + props.oamImagery.url + : "Bing" + }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ + layer.aoiId + }`, + "_blank", + "noreferrer" + ); + }} + > + {/* */} + RapiD logo + + + + { + // mutateFetch(layer.aoiId); + // console.log("Open in Editor") + window.open( + `https://www.openstreetmap.org/edit/#background=${ + props.oamImagery + ? "custom:" + props.oamImagery.url + : "Bing" + }&disable_features=boundaries&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ + layer.aoiId + }&map=10.70/18.9226/81.6991`, + "_blank", + "noreferrer" + ); + }} + > + {/* */} + OSM logo + + + + { + mutateFetch(layer.aoiId); + console.log("call raw data API to fetch OSM labels"); + }} + > + + + + {/* { + + console.log("Remove labels ") + }}> + + */} + + { + const lat = + layer.latlngs.reduce(function ( + accumulator, + curValue + ) { + return accumulator + curValue.lat; + }, + 0) / layer.latlngs.length; + const lng = + layer.latlngs.reduce(function ( + accumulator, + curValue + ) { + return accumulator + curValue.lng; + }, + 0) / layer.latlngs.length; + // [lat, lng] are the centroid of the polygon + props.selectAOIHandler([lat, lng], 17); + }} + > + + + + + { + mutateDeleteAOI(layer.id); + }} + > + + + + + + ))} + + + {props.mapLayers && props.mapLayers.length === 0 && ( + + No AOIs yet, start creating one by selecting AOIs on the top and + create a polygon + + )} + + + ); +}; +export default FeedbackAOI; diff --git a/frontend/src/components/Layout/Feedback/FeedbackAOIDetails.js b/frontend/src/components/Layout/Feedback/FeedbackAOIDetails.js new file mode 100644 index 00000000..50d6dfa2 --- /dev/null +++ b/frontend/src/components/Layout/Feedback/FeedbackAOIDetails.js @@ -0,0 +1,50 @@ +import { Typography } from "@material-ui/core"; +import React from "react"; +import { useQuery } from "react-query"; + +import axios from "../../../axios"; + +import { timeSince, aoiStatusText } from "../../../utils"; +const FeedbackAOIDetails = (props) => { + // console.log("rendering AOIDetails",props) + const fetchAOI = async () => { + try { + const res = await axios.get(`/feedback-aoi/${props.id}/`); + + if (res.error) { + // setMapError(res.error.response.statusText); + console.log(res.error.response.statusText); + } else { + // success full fetch + // console.log("API details, ",props.aoiId,res.data); + return res.data; + } + } catch (e) { + console.log("isError", e); + } finally { + } + }; + const { data } = useQuery("fetchAOI" + props.id, fetchAOI, { + refetchInterval: 5000, + }); + + return ( + <> + {data && ( + + {aoiStatusText(data.properties.label_status)}{" "} + {data.properties.label_fetched && + timeSince(new Date(data.properties.label_fetched), new Date())} + + )} + + ); +}; + +export default FeedbackAOIDetails; diff --git a/frontend/src/components/Layout/Feedback/Pagination.js b/frontend/src/components/Layout/Feedback/Pagination.js new file mode 100644 index 00000000..24dfbaeb --- /dev/null +++ b/frontend/src/components/Layout/Feedback/Pagination.js @@ -0,0 +1,29 @@ +import React, { useState } from "react"; + +function usePagination(data, itemsPerPage) { + const [currentPage, setCurrentPage] = useState(1); + const maxPage = Math.ceil(data.length / itemsPerPage); + + function currentData() { + const begin = (currentPage - 1) * itemsPerPage; + const end = begin + itemsPerPage; + return data.slice(begin, end); + } + + function next() { + setCurrentPage((currentPage) => Math.min(currentPage + 1, maxPage)); + } + + function prev() { + setCurrentPage((currentPage) => Math.max(currentPage - 1, 1)); + } + + function jump(page) { + const pageNumber = Math.max(1, page); + setCurrentPage((currentPage) => Math.min(pageNumber, maxPage)); + } + + return { next, prev, jump, currentData, currentPage, maxPage }; +} + +export default usePagination; diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css index 5c0c8ca8..0b4bb2c4 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.css @@ -50,14 +50,6 @@ margin-top: -20px; } -.card -{ - border: 2px solid ; - border-radius: 10px; - box-shadow: 5px 5px 5px #cdcdcd; - margin-bottom: 5px; - padding: 10px 5px; -} .small-checkbox { diff --git a/frontend/src/index.css b/frontend/src/index.css index a632d284..38b7bab8 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -151,3 +151,17 @@ code { { padding: 1px; } + +.card +{ + border: 2px solid ; + border-radius: 10px; + box-shadow: 5px 5px 5px #cdcdcd; + margin-bottom: 5px; + padding: 10px 5px; +} + +.margin-left-12 +{ + margin-left: 12px !important; +} \ No newline at end of file diff --git a/frontend/src/routes.js b/frontend/src/routes.js index 4cafd3bc..f4e26b4d 100644 --- a/frontend/src/routes.js +++ b/frontend/src/routes.js @@ -13,7 +13,9 @@ const Documentation = React.lazy(() => ); const Learn = React.lazy(() => import("./components/Layout/Learn/Learn")); const Start = React.lazy(() => import("./components/Layout/Start/Start")); -const Why = React.lazy(() => import("./components/Layout/Why/Why")); +const Feedback = React.lazy(() => + import("./components/Layout/Feedback/Feedback") +); export const publicRoutes = [ // add here all route you wish to implement and associate each one with a component, recommended to use lazy loading @@ -23,12 +25,13 @@ export const publicRoutes = [ element: , }, { path: "/ai-models/*", name: "Training Datasets", element: }, + { path: "/documentation", name: "Training Datasets", element: , }, - { path: "/learn/*", name: "Learn", element: }, + { path: "/learn", name: "Learn", element: }, { path: "/start-mapping/*", name: "Training Datasets", element: }, // { path: "/why-fair", name: "Training Datasets", element: }, { path: "/authenticate", name: "Authenticate", element: }, From 9e11ebc88c34e0836e57247a0ac71a5fdaa1b884 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Fri, 11 Aug 2023 10:55:27 +0545 Subject: [PATCH 03/17] Added api for feedback labela and osm fetch --- backend/core/admin.py | 13 ++++++++- backend/core/serializers.py | 8 ++---- backend/core/urls.py | 9 ++++-- backend/core/utils.py | 55 ++++++++++++++++++++++++------------- backend/core/views.py | 48 ++++++++++++++++++++++++++++++-- 5 files changed, 103 insertions(+), 30 deletions(-) diff --git a/backend/core/admin.py b/backend/core/admin.py index ea57bef1..e6daf9dd 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -13,7 +13,7 @@ class DatasetAdmin(geoadmin.GeoModelAdmin): @admin.register(Model) class ModelAdmin(geoadmin.GeoModelAdmin): - list_display = ["get_dataset_id", "name", "status", "created_at"] + list_display = ["get_dataset_id", "name", "status", "created_at", "created_by"] def get_dataset_id(self, obj): return obj.dataset.id @@ -37,3 +37,14 @@ def get_model_id(self, obj): return obj.model.id get_model_id.short_description = "Model" + + +# dsaf +@admin.register(FeedbackAOI) +class FeedbackAOIAdmin(geoadmin.GeoModelAdmin): + list_display = ["training", "user"] + + +@admin.register(FeedbackLabel) +class FeedbackLabelAdmin(geoadmin.GeoModelAdmin): + list_display = ["feedback_aoi", "created_at"] diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 2381ce3e..89066224 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -102,14 +102,12 @@ def to_representation(self, instance): return ret -class LabelSerializer( - GeoFeatureModelSerializer -): # serializers are used to translate models objects to api +class LabelSerializer(GeoFeatureModelSerializer): class Meta: model = Label - geo_field = "geom" # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson + geo_field = "geom" # auto_bbox = True - fields = "__all__" # defining all the fields to be included in curd for now , we can restrict few if we want + fields = "__all__" read_only_fields = ("created_at", "osm_id") diff --git a/backend/core/urls.py b/backend/core/urls.py index 8c395cdd..d1466c2a 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -15,7 +15,8 @@ LabelViewSet, ModelViewSet, PredictionView, - RawdataApiView, + RawdataApiAOIView, + RawdataApiFeedbackView, TrainingViewSet, TrainingWorkspaceDownloadView, TrainingWorkspaceView, @@ -39,7 +40,11 @@ urlpatterns = [ path("", include(router.urls)), - path("label/osm/fetch//", RawdataApiView.as_view()), + path("label/osm/fetch//", RawdataApiAOIView.as_view()), + path( + "label/feedback/osm/fetch//", + RawdataApiFeedbackView.as_view(), + ), # path("download//", download_training_data), path("training/status//", run_task_status), path("training/publish//", publish_training), diff --git a/backend/core/utils.py b/backend/core/utils.py index 11c59186..58dcd259 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -10,8 +10,8 @@ from django.conf import settings from tqdm import tqdm -from .models import AOI, Label -from .serializers import LabelSerializer +from .models import AOI, FeedbackAOI, FeedbackLabel, Label +from .serializers import FeedbackLabelSerializer, LabelSerializer def get_dir_size(directory): @@ -79,7 +79,6 @@ def latlng2tile(zoom, lat, lng, tile_size): def get_start_end_download_coords(bbox_coords, zm_level, tile_size): - # start point where we will start downloading the tiles start_point_lng = bbox_coords[0] # getting the starting lat lng @@ -205,7 +204,7 @@ def request_rawdata(request_params): return response_back -def process_rawdata(file_download_url, aoi_id): +def process_rawdata(file_download_url, aoi_id, feedback=False): """This will create temp directory , Downloads file from URL provided, Unzips it Finds a geojson file , Process it and finally removes processed Geojson file and downloaded zip file from Directory""" @@ -234,7 +233,7 @@ def process_rawdata(file_download_url, aoi_id): print(f"""Geojson file{fileName} from API wrote to disk""") break geojson_file = f"""{geojson_file_path}{fileName}""" - process_geojson(geojson_file, aoi_id) + process_geojson(geojson_file, aoi_id, feedback) remove_file(file_temp_path) remove_file(geojson_file) @@ -244,28 +243,41 @@ def remove_file(path: str) -> None: os.unlink(path) -def process_feature(feature, aoi_id, dataset_id): +def process_feature(feature, aoi_id, foreign_key_id, feedback=False): """Multi thread process of features""" properties = feature["properties"] osm_id = properties["osm_id"] geometry = feature["geometry"] + if feedback: + if FeedbackLabel.objects.filter( + osm_id=int(osm_id), feedback_aoi__training=foreign_key_id + ).exists(): + FeedbackLabel.objects.filter( + osm_id=int(osm_id), feedback_aoi__training=foreign_key_id + ).delete() + + label = FeedbackLabelSerializer( + data={"osm_id": int(osm_id), "geom": geometry, "feedback_aoi": aoi_id} + ) - if Label.objects.filter(osm_id=int(osm_id), aoi__dataset=dataset_id).exists(): - - Label.objects.filter(osm_id=int(osm_id), aoi__dataset=dataset_id).delete() - # print(f"Existing record Found and Dropped {osm_id}") - - label = LabelSerializer( - data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id} - ) + else: + if Label.objects.filter( + osm_id=int(osm_id), aoi__dataset=foreign_key_id + ).exists(): + Label.objects.filter( + osm_id=int(osm_id), aoi__dataset=foreign_key_id + ).delete() + + label = LabelSerializer( + data={"osm_id": int(osm_id), "geom": geometry, "aoi": aoi_id} + ) if label.is_valid(): - label.save() # update if it exists create if not + label.save() else: raise ValidationErr(label.errors) - # print(f"Created {osm_id}") -def process_geojson(geojson_file_path, aoi_id): +def process_geojson(geojson_file_path, aoi_id, feedback=False): """Responsible for Processing Geojson file from directory , Opens the file reads the record , Checks either record present or not if not inserts into database @@ -278,7 +290,10 @@ def process_geojson(geojson_file_path, aoi_id): ValidationErr: _description_ """ print("Geojson Processing Started") - dataset_id = AOI.objects.get(id=aoi_id).dataset + if feedback: + foreign_key_id = FeedbackAOI.objects.get(id=aoi_id).training + else: + foreign_key_id = AOI.objects.get(id=aoi_id).dataset max_workers = ( (os.cpu_count() - 1) if os.cpu_count() != 1 else 1 ) # leave one cpu free always @@ -289,7 +304,9 @@ def process_geojson(geojson_file_path, aoi_id): data = json.load(f) with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [ - executor.submit(process_feature, feature, aoi_id, dataset_id) + executor.submit( + process_feature, feature, aoi_id, foreign_key_id, feedback + ) for feature in data["features"] ] for f in tqdm(futures, total=len(data["features"])): diff --git a/backend/core/views.py b/backend/core/views.py index 500df8d9..b56ba774 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -196,7 +196,12 @@ class FeedbackLabelViewset(viewsets.ModelViewSet): queryset = FeedbackLabel.objects.all() http_method_names = ["get", "post", "patch", "delete"] serializer_class = FeedbackLabelSerializer - filterset_fields = ["feedback_aoi"] + bbox_filter_field = "geom" + filter_backends = ( + InBBoxFilter, # it will take bbox like this api/v1/label/?in_bbox=-90,29,-89,35 , + ) + bbox_filter_include_overlapping = True + filterset_fields = ["feedback_aoi", "feedback_aoi__training"] class ModelViewSet( @@ -238,12 +243,49 @@ class LabelViewSet(viewsets.ModelViewSet): filterset_fields = ["aoi", "aoi__dataset"] -class RawdataApiView(APIView): +class RawdataApiFeedbackView(APIView): + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] + + def post(self, request, feedbackaoi_id, *args, **kwargs): + """Downloads available osm data as labels within given feedback aoi + + Args: + request (_type_): _description_ + feedbackaoi_id (_type_): _description_ + + Returns: + status: Success/Failed + """ + obj = get_object_or_404(FeedbackAOI, id=feedbackaoi_id) + try: + obj.label_status = 0 + obj.save() + raw_data_params = { + "geometry": json.loads(obj.geom.geojson), + "filters": {"tags": {"polygon": {"building": []}}}, + "geometryType": ["polygon"], + } + result = request_rawdata(raw_data_params) + file_download_url = result["download_url"] + process_rawdata(file_download_url, feedbackaoi_id, feedback=True) + obj.label_status = 1 + obj.label_fetched = datetime.utcnow() + obj.save() + return Response("Success", status=status.HTTP_201_CREATED) + except Exception as ex: + obj.label_status = -1 + obj.save() + # raise ex + return Response("OSM Fetch Failed", status=500) + + +class RawdataApiAOIView(APIView): authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] def post(self, request, aoi_id, *args, **kwargs): - """Downloads available osm data as labels within given aoi + """Downloads available osm data as labels within given feedback Args: request (_type_): _description_ From 4aaf36ac3ce5dc451f5b297714c652eabda26019 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 14:34:24 +0545 Subject: [PATCH 04/17] Fix bugs on Feedback label overwrite --- backend/core/admin.py | 15 +++++++-------- backend/core/serializers.py | 4 ++-- backend/core/views.py | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/backend/core/admin.py b/backend/core/admin.py index e6daf9dd..ccd5af55 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -7,12 +7,12 @@ @admin.register(Dataset) -class DatasetAdmin(geoadmin.GeoModelAdmin): +class DatasetAdmin(geoadmin.OSMGeoAdmin): list_display = ["name", "created_by"] @admin.register(Model) -class ModelAdmin(geoadmin.GeoModelAdmin): +class ModelAdmin(geoadmin.OSMGeoAdmin): list_display = ["get_dataset_id", "name", "status", "created_at", "created_by"] def get_dataset_id(self, obj): @@ -22,7 +22,7 @@ def get_dataset_id(self, obj): @admin.register(Training) -class TrainingAdmin(geoadmin.GeoModelAdmin): +class TrainingAdmin(geoadmin.OSMGeoAdmin): list_display = [ "get_model_id", "description", @@ -39,12 +39,11 @@ def get_model_id(self, obj): get_model_id.short_description = "Model" -# dsaf @admin.register(FeedbackAOI) -class FeedbackAOIAdmin(geoadmin.GeoModelAdmin): +class FeedbackAOIAdmin(geoadmin.OSMGeoAdmin): list_display = ["training", "user"] -@admin.register(FeedbackLabel) -class FeedbackLabelAdmin(geoadmin.GeoModelAdmin): - list_display = ["feedback_aoi", "created_at"] +@admin.register(Feedback) +class FeedbackAdmin(geoadmin.OSMGeoAdmin): + list_display = ["feedback_type", "training", "user", "created_at"] diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 89066224..223413e5 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -109,7 +109,7 @@ class Meta: # auto_bbox = True fields = "__all__" - read_only_fields = ("created_at", "osm_id") + # read_only_fields = ("created_at", "osm_id") class FeedbackLabelSerializer(GeoFeatureModelSerializer): @@ -117,7 +117,7 @@ class Meta: model = FeedbackLabel geo_field = "geom" fields = "__all__" - read_only_fields = ("created_at", "osm_id") + # read_only_fields = ("created_at", "osm_id") class LabelFileSerializer( diff --git a/backend/core/views.py b/backend/core/views.py index b56ba774..765f1485 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet): class RawdataApiFeedbackView(APIView): - authentication_classes = [OsmAuthentication] - permission_classes = [IsOsmAuthenticated] + # authentication_classes = [OsmAuthentication] + # permission_classes = [IsOsmAuthenticated] def post(self, request, feedbackaoi_id, *args, **kwargs): """Downloads available osm data as labels within given feedback aoi From c7c9e76ddd6a5d2e069f7259998d821b8404f9c5 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Mon, 14 Aug 2023 16:37:01 +0200 Subject: [PATCH 05/17] fix learn section refresh error --- frontend/public/hot-marker.png | Bin 0 -> 2934 bytes .../public/{learn => learn-resources}/1.png | Bin .../public/{learn => learn-resources}/10.png | Bin .../public/{learn => learn-resources}/11.png | Bin .../public/{learn => learn-resources}/12.png | Bin .../public/{learn => learn-resources}/13.png | Bin .../public/{learn => learn-resources}/14.png | Bin .../public/{learn => learn-resources}/15.png | Bin .../public/{learn => learn-resources}/16.png | Bin .../public/{learn => learn-resources}/17.png | Bin .../public/{learn => learn-resources}/18.png | Bin .../public/{learn => learn-resources}/19.png | Bin .../public/{learn => learn-resources}/2.png | Bin .../public/{learn => learn-resources}/20.png | Bin .../public/{learn => learn-resources}/21.png | Bin .../public/{learn => learn-resources}/22.png | Bin .../public/{learn => learn-resources}/3.png | Bin .../public/{learn => learn-resources}/4.png | Bin .../public/{learn => learn-resources}/5.png | Bin .../public/{learn => learn-resources}/6.png | Bin .../public/{learn => learn-resources}/7.png | Bin .../public/{learn => learn-resources}/8.png | Bin .../public/{learn => learn-resources}/9.png | Bin .../public/{learn => learn-resources}/osm.png | Bin frontend/src/components/Layout/Learn/Learn.js | 30 +++++++++--------- 25 files changed, 15 insertions(+), 15 deletions(-) create mode 100644 frontend/public/hot-marker.png rename frontend/public/{learn => learn-resources}/1.png (100%) rename frontend/public/{learn => learn-resources}/10.png (100%) rename frontend/public/{learn => learn-resources}/11.png (100%) rename frontend/public/{learn => learn-resources}/12.png (100%) rename frontend/public/{learn => learn-resources}/13.png (100%) rename frontend/public/{learn => learn-resources}/14.png (100%) rename frontend/public/{learn => learn-resources}/15.png (100%) rename frontend/public/{learn => learn-resources}/16.png (100%) rename frontend/public/{learn => learn-resources}/17.png (100%) rename frontend/public/{learn => learn-resources}/18.png (100%) rename frontend/public/{learn => learn-resources}/19.png (100%) rename frontend/public/{learn => learn-resources}/2.png (100%) rename frontend/public/{learn => learn-resources}/20.png (100%) rename frontend/public/{learn => learn-resources}/21.png (100%) rename frontend/public/{learn => learn-resources}/22.png (100%) rename frontend/public/{learn => learn-resources}/3.png (100%) rename frontend/public/{learn => learn-resources}/4.png (100%) rename frontend/public/{learn => learn-resources}/5.png (100%) rename frontend/public/{learn => learn-resources}/6.png (100%) rename frontend/public/{learn => learn-resources}/7.png (100%) rename frontend/public/{learn => learn-resources}/8.png (100%) rename frontend/public/{learn => learn-resources}/9.png (100%) rename frontend/public/{learn => learn-resources}/osm.png (100%) diff --git a/frontend/public/hot-marker.png b/frontend/public/hot-marker.png new file mode 100644 index 0000000000000000000000000000000000000000..d052f4e2e3c9180921f48a840f23c70d0af1d3f7 GIT binary patch literal 2934 zcmV-+3yJiJP)E&G_}0t&=HK{qgMbxTx{6q4Tz&@S}<4UohcE6z-dX=W$Q@-`MDORr}@L=yXfuS0(3aKkJHa z=z3h^RxRR99_3^}^un|8te5J9X6%u7^S!X`k#XyVVDF@m>yLQkT{q`uHRo+Z?3I7; zs+s0rE8FW>)LJ)p-6Fh)|^bo2MhS95>TwN28bg#rVJptVgwSspNC~bK{i5S~UKB>p8&;_no&tp?}z^cf&Z>GW|FjT_0A1Kmrt( z5g^?F_8xBkC|v z2xl3YQ~g%oguvNIb%&Yw7)ux0=FkG2tqZ0;!Gl`SXlEUcH*fOys@yw3rAHFaNoNZDs ztSCHim;}yU4I2*QldFBRy&ieV|0zaomRaUg;*k4OjQ3du7NI@n73e>EjAL|;WM23i zm%JvNOB}?E0^PsIIMNM_2(<$Dm+JU?A=C1>aP#3`@0cW7N+`Axi;6lntA3CYxTU(mEkMv1odjk_M>=I*8f5{y>rFTpP}R-%tjri_6A0_6RX8`C>Lc z98L(ox7$HZ?TuUfQ-x=y0e2dRs9%IkN8$KDv7O-K0kOAZM;tSNp*B|Vo+=>okc zg-c4}uY`|mLEq$u7^2q-YV@y$k4{RHIn!<*`yOyWjEs%Yv&qTj@N_wtbZ@Gkb*@M_M#D0Q$CO`x!QJVbuuCIeFCzsQ%6w+c^>utp5bO{Ot+ zy?1vMo?iM7`Sf6-u7hQwV!%3b7d<0h)Dv zv5R{Hd8oJ~Pmvq9K(DNC{6Z=mD(0i>*acCqtZ$s*eq%ZL@d_Vvs}MU%4-~({5q49k zSJpR>GYuT6DfFATF=BAFK)Q#}R$DEwQ6T7R*7XH;Pvfe&H}L@ZonL9z^+gZq9=d9| zh42;k9U9zuW4$NmQOURHA|r}{iuZ}5e1+T^_f;Y(eiP}ssDwz6RFDf|(N%FSN?6yz zV4MaWYjb`pJUvEc4CIx;Ts}5lDx^Y46S>+J-HyRp?;bi7ML*i(V=Kg5lj$|=idBRY z^41jTd=}x(WM;;1VTY;c5FZ_4QzFuA1x+2BKY?S~bn0JYGm}osXoj@Ak4~@A5}EY} z>`!-Mwji-`jm;2&cR`r)y6E(w;XyicFRJKVZ5tnakI&HAb9rM>-JCAIoD0yWCdwUj zg%K@zu7dvue`*4%xVI@k<4dUI#Q5RFCgEU+^e`=hV!Q>5f}uDld?6hBf@*W)Pu?2@ z_+wUZ%1|+TMmb9e?d$Z2hF?doKX1<)m={=Rv?<3gp?IHG%v0-N*TFYLk)_h|l~gbz z;O=qV$f`UM575FMhrq%PvD^Zr^EXPcW4v3Xwml1;SQM7XjWC1^`kh7?mrK6y&wG8A z5RQwv5Ea>^&L(7We1qCDs^DlD`} zl`}dJUW-#8#SzwoWNtI))5@Q=M|j)~Tq5SC3M`J{A_G>1^a%6zL_9sB{sAYf6X`pT zoLe`hBJM?cj6H@xaP~F%gmM4vp4kvsh?E-#P~mHcv4}t5Y5?LsTk6yvne6NN4YSp1 zX_U`+jObFbSbB*F5q@>x)n2dN$}_|MTY%ZJ*mS537E~}@yq!+v+PK1*Q(^VW3-f*3 zz^VynYa%HVrGO}!E91gp+QoolTm5H0IX63aQc}VX(|kN zSur7{sS5?(#=#qy8dx#lI6Ees*O?noILO3W_<{a=AmG9F^(%|`2ig@r+Sas0Lvub+ zKqjtw`i|xpKilyqJUy+)Gt~OO80K8(-p%Dth;F8*aZQANM(&hb0No-80bhT45?yUx zl(ZPfPeWh3eb;R9mO8qQ5y^cE#}aU~w0Ke6=esEAInXEXGi$;^VXWbbB#y!#bz(}% z&>G++gWH)ep2$u2S4_xqIes+pKWL=#rU)BR4fuJ#rWYRCtu;59a7EyKNDI%-$kK<><_qL z!iNkcL^ngN*)Liu8rcH!f&nrIJJRovjcBl;r5P&d`&#~)(96wQ_yI8h?w)5f%)Oo( zDT^Tl@2%T9DBVzu=$LQ+q@RfJB?`#W*%>UAXS@?-_NFmoG|&x_^5ZrwQbu1f&s}S; zjVaQJ62ew};kIB~pqTFzXp9S7o(3j4HQ{X4N0qigJD>B+8>oUBIL&yrDUB}XD45KX zZ@(A{W8D{2t(BtdVdGqrw*b@%@rHUD;pB8n z`pKrED0!Ge*LRT(R|seJ7$TlH^lUYARC>^bWvID;I4Q9dsZo}Or`>U(4MW3bl){pb9JnH zGmRC6O82j4m}q&X$wj3hg=jh6Cjb60h27=7xAU0eQ#47eDm?#D+tQgr;W`ThCU3d7 gLHKK#22(Wne;O{mC#%dsMF0Q*07*qoM6N<$g0U~T!~g&Q literal 0 HcmV?d00001 diff --git a/frontend/public/learn/1.png b/frontend/public/learn-resources/1.png similarity index 100% rename from frontend/public/learn/1.png rename to frontend/public/learn-resources/1.png diff --git a/frontend/public/learn/10.png b/frontend/public/learn-resources/10.png similarity index 100% rename from frontend/public/learn/10.png rename to frontend/public/learn-resources/10.png diff --git a/frontend/public/learn/11.png b/frontend/public/learn-resources/11.png similarity index 100% rename from frontend/public/learn/11.png rename to frontend/public/learn-resources/11.png diff --git a/frontend/public/learn/12.png b/frontend/public/learn-resources/12.png similarity index 100% rename from frontend/public/learn/12.png rename to frontend/public/learn-resources/12.png diff --git a/frontend/public/learn/13.png b/frontend/public/learn-resources/13.png similarity index 100% rename from frontend/public/learn/13.png rename to frontend/public/learn-resources/13.png diff --git a/frontend/public/learn/14.png b/frontend/public/learn-resources/14.png similarity index 100% rename from frontend/public/learn/14.png rename to frontend/public/learn-resources/14.png diff --git a/frontend/public/learn/15.png b/frontend/public/learn-resources/15.png similarity index 100% rename from frontend/public/learn/15.png rename to frontend/public/learn-resources/15.png diff --git a/frontend/public/learn/16.png b/frontend/public/learn-resources/16.png similarity index 100% rename from frontend/public/learn/16.png rename to frontend/public/learn-resources/16.png diff --git a/frontend/public/learn/17.png b/frontend/public/learn-resources/17.png similarity index 100% rename from frontend/public/learn/17.png rename to frontend/public/learn-resources/17.png diff --git a/frontend/public/learn/18.png b/frontend/public/learn-resources/18.png similarity index 100% rename from frontend/public/learn/18.png rename to frontend/public/learn-resources/18.png diff --git a/frontend/public/learn/19.png b/frontend/public/learn-resources/19.png similarity index 100% rename from frontend/public/learn/19.png rename to frontend/public/learn-resources/19.png diff --git a/frontend/public/learn/2.png b/frontend/public/learn-resources/2.png similarity index 100% rename from frontend/public/learn/2.png rename to frontend/public/learn-resources/2.png diff --git a/frontend/public/learn/20.png b/frontend/public/learn-resources/20.png similarity index 100% rename from frontend/public/learn/20.png rename to frontend/public/learn-resources/20.png diff --git a/frontend/public/learn/21.png b/frontend/public/learn-resources/21.png similarity index 100% rename from frontend/public/learn/21.png rename to frontend/public/learn-resources/21.png diff --git a/frontend/public/learn/22.png b/frontend/public/learn-resources/22.png similarity index 100% rename from frontend/public/learn/22.png rename to frontend/public/learn-resources/22.png diff --git a/frontend/public/learn/3.png b/frontend/public/learn-resources/3.png similarity index 100% rename from frontend/public/learn/3.png rename to frontend/public/learn-resources/3.png diff --git a/frontend/public/learn/4.png b/frontend/public/learn-resources/4.png similarity index 100% rename from frontend/public/learn/4.png rename to frontend/public/learn-resources/4.png diff --git a/frontend/public/learn/5.png b/frontend/public/learn-resources/5.png similarity index 100% rename from frontend/public/learn/5.png rename to frontend/public/learn-resources/5.png diff --git a/frontend/public/learn/6.png b/frontend/public/learn-resources/6.png similarity index 100% rename from frontend/public/learn/6.png rename to frontend/public/learn-resources/6.png diff --git a/frontend/public/learn/7.png b/frontend/public/learn-resources/7.png similarity index 100% rename from frontend/public/learn/7.png rename to frontend/public/learn-resources/7.png diff --git a/frontend/public/learn/8.png b/frontend/public/learn-resources/8.png similarity index 100% rename from frontend/public/learn/8.png rename to frontend/public/learn-resources/8.png diff --git a/frontend/public/learn/9.png b/frontend/public/learn-resources/9.png similarity index 100% rename from frontend/public/learn/9.png rename to frontend/public/learn-resources/9.png diff --git a/frontend/public/learn/osm.png b/frontend/public/learn-resources/osm.png similarity index 100% rename from frontend/public/learn/osm.png rename to frontend/public/learn-resources/osm.png diff --git a/frontend/src/components/Layout/Learn/Learn.js b/frontend/src/components/Layout/Learn/Learn.js index cdae6135..60ef63db 100644 --- a/frontend/src/components/Layout/Learn/Learn.js +++ b/frontend/src/components/Layout/Learn/Learn.js @@ -23,105 +23,105 @@ const Learn = () => { title: "Step 1: First Login & Click on the button Start Creating Dataset", description: "To create a new training dataset, start by clicking on the 'Start Creating Dataset' button.", - image: "/learn/1.png", + image: "/learn-resources/1.png", }, { id: 2, title: "Step 2: Click on the button Create New", description: "After clicking on 'Start Creating Dataset', click on the 'Create New' button to create a new dataset.", - image: "/learn/2.png", + image: "/learn-resources/2.png", }, { id: 3, title: "Step 3: Click on the input field , Give your Dataset Name", description: "To name your dataset, click on the input field and type in your desired name & click on the 'Create Training Dataset'", - image: "/learn/3.png", + image: "/learn-resources/3.png", }, { id: 4, title: "Step 4: Find Image to Train", description: "After creating Dataset You will see following screen , Now Open Open Aerial Map in New Tab , https://openaerialmap.org/", - image: "/learn/4.png", + image: "/learn-resources/4.png", }, { id: 5, title: "Step 5: Find Drone Image and Copy TMS URL", description: "Look for your area and find good Drone Image for training . After finding , Click on Copy Image URL TMS", - image: "/learn/5.png", + image: "/learn-resources/5.png", }, { id: 6, title: "Step 6: Paste TMS To Open Aerial Imagery Tab", description: "Paste your TMS URL that you have copied from Open Aerial Map Drone Image", - image: "/learn/6.png", + image: "/learn-resources/6.png", }, { id: 7, title: "Step 7: Zoom to Layer and Visualize Image", description: "Click on Zoom to layer next to your Image name in OAM Block Top Right side of screen", - image: "/learn/7.png", + image: "/learn-resources/7.png", }, { id: 8, title: "Step 8: Create Area of Interest for Training", description: "Click on top left map buttons below zoom , To create AOI , AOI will be used to create labels. Labels are Buildings that will be acting as input for model inside AOI", - image: "/learn/8.png", + image: "/learn-resources/8.png", }, { id: 9, title: "Step 9: Fetch Exisiting OSM Buildings in your Area of Interest", description: "Click on Fetch OSM Data button next to OSM Logo Inside List of Area of Interest on Right side.", - image: "/learn/9.png", + image: "/learn-resources/9.png", }, { id: 10, title: "Step 10: Visualize each Buildings to check their accuracy", description: "Zoom to Level 20 in Map to see OSM Buildings that you have just fetched", - image: "/learn/10.png", + image: "/learn-resources/10.png", }, { id: 11, title: "Step 11: Correct your labels", description: "For training it is crucial that buildings are aligned exactly on Drone image, More good your input is more good your output will be. If you want to edit it Click on OSM Logo and Fix buildings / labels in your AOI", - image: "/learn/11.png", + image: "/learn-resources/11.png", }, { id: 12, title: "Step 12: Upload your fixes", description: "You should see your AOI in ID , Fix your labels and Upload your changes to OSM", - image: "/learn/osm.png", + image: "/learn-resources/osm.png", }, { id: 13, title: "Step 13: Come Back to fAIr , Fetch OSM Labels", description: "After you do your changes on OSM , Comeback to fAIr and fetch new labels, Give it a few min to update our database ( Click on button next to OSM Logo on AOI to fetch -Step 9)", - image: "/learn/12.png", + image: "/learn-resources/12.png", }, { id: 14, title: "Step 14: Create Model for you Dataset", description: "Click on View Models on your dataset page , You will see this page. Click on Create New", - image: "/learn/13.png", + image: "/learn-resources/13.png", }, { id: 15, title: "Step 15: Provide Model Metadata", description: "Give your model name , Choose Your Dataset from Dropdown , Select BaseModel : RAMP is default for now , Click on Create AI Model", - image: "/learn/14.png", + image: "/learn-resources/14.png", }, { id: 16, From 9e0a3510dd92ca0a5b2053f51bcef03ed372ffb5 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 20:42:18 +0545 Subject: [PATCH 06/17] Implemented API for apply/feedback --- backend/core/serializers.py | 28 +++++++++-- backend/core/tasks.py | 94 +++++++++++++------------------------ backend/core/views.py | 22 +++++---- docker-compose.yml | 4 +- 4 files changed, 70 insertions(+), 78 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 223413e5..9d655aca 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -120,12 +120,18 @@ class Meta: # read_only_fields = ("created_at", "osm_id") -class LabelFileSerializer( - GeoFeatureModelSerializer -): # serializers are used to translate models objects to api +class LabelFileSerializer(GeoFeatureModelSerializer): class Meta: model = Label - geo_field = "geom" # this will be used as geometry in order to create geojson api , geofeatureserializer will let you create api in geojson + geo_field = "geom" + # auto_bbox = True + fields = ("osm_id",) + + +class FeedbackLabelFileSerializer(GeoFeatureModelSerializer): + class Meta: + model = FeedbackLabel + geo_field = "geom" # auto_bbox = True fields = ("osm_id",) @@ -158,7 +164,19 @@ class FeedbackParamSerializer(serializers.Serializer): training_id = serializers.IntegerField(required=True) epochs = serializers.IntegerField(required=False) batch_size = serializers.IntegerField(required=False) - freeze_layers = serializers.BooleanField(required=False) + zoom_level = serializers.ListField(required=False) + + def validate(self, data): + """ + Check supplied data + """ + if "zoom_level" in data: + for i in data["zoom_level"]: + if int(i) < 19 or int(i) > 21: + raise serializers.ValidationError( + "Zoom level Supported between 19-21" + ) + return data class PredictionParamSerializer(serializers.Serializer): diff --git a/backend/core/tasks.py b/backend/core/tasks.py index 8f020b81..4cd34642 100644 --- a/backend/core/tasks.py +++ b/backend/core/tasks.py @@ -10,8 +10,12 @@ import ramp.utils import tensorflow as tf from celery import shared_task -from core.models import AOI, Feedback, Label, Training -from core.serializers import FeedbackFileSerializer, LabelFileSerializer +from core.models import AOI, Feedback, FeedbackAOI, FeedbackLabel, Label, Training +from core.serializers import ( + FeedbackFileSerializer, + FeedbackLabelFileSerializer, + LabelFileSerializer, +) from core.utils import bbox, download_imagery, get_start_end_download_coords from django.conf import settings from django.contrib.gis.db.models.aggregates import Extent @@ -65,34 +69,32 @@ def train_model( shutil.rmtree(training_input_image_source) os.makedirs(training_input_image_source) if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - bbox_feedback = feedback_objects.aggregate(Extent("geom"))[ - "geom__extent" - ] - bbox_geo = GEOSGeometry( - f"POLYGON(({bbox_feedback[0]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[1]},{bbox_feedback[2]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[3]},{bbox_feedback[0]} {bbox_feedback[1]}))" - ) - print(training_input_image_source) - print(bbox_feedback) - with open( - os.path.join(training_input_image_source, "labels_bbox.geojson"), - "w", - encoding="utf-8", - ) as f: - f.write(bbox_geo.geojson) + try: + aois = FeedbackAOI.objects.filter(training=feedback) + except FeedbackAOI.DoesNotExist: + raise ValueError( + f"No Feedback AOI is attached with supplied training id:{dataset_id}, Create AOI first", + ) + + else: + try: + aois = AOI.objects.filter(dataset=dataset_id) + except AOI.DoesNotExist: + raise ValueError( + f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", + ) + for obj in aois: + bbox_coords = bbox(obj.geom.coords[0]) for z in zoom_level: zm_level = z print( f"""Running Download process for - feedback {training_id} - dataset : {dataset_id} , zoom : {zm_level}""" + aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" ) try: tile_size = DEFAULT_TILE_SIZE # by default - bbox_coords = list(bbox_feedback) + start, end = get_start_end_download_coords( bbox_coords, zm_level, tile_size ) @@ -107,49 +109,17 @@ def train_model( except Exception as ex: raise ex - else: - try: - aois = AOI.objects.filter(dataset=dataset_id) - except AOI.DoesNotExist: - raise ValueError( - f"No AOI is attached with supplied dataset id:{dataset_id}, Create AOI first", - ) - - for obj in aois: - bbox_coords = bbox(obj.geom.coords[0]) - for z in zoom_level: - zm_level = z - print( - f"""Running Download process for - aoi : {obj.id} - dataset : {dataset_id} , zoom : {zm_level}""" - ) - try: - tile_size = DEFAULT_TILE_SIZE # by default - - start, end = get_start_end_download_coords( - bbox_coords, zm_level, tile_size - ) - # start downloading - download_imagery( - start, - end, - zm_level, - base_path=training_input_image_source, - source=source_imagery, - ) - except Exception as ex: - raise ex - ## -----------LABEL GENERATOR--------- - logging.debug("Label Generator started") + logging.info("Label Generator started") + aoi_list = [r.id for r in aois] + logging.info(aoi_list) + if feedback: - feedback_objects = Feedback.objects.filter( - training__id=feedback, - validated=True, - ) - serialized_field = FeedbackFileSerializer(feedback_objects, many=True) + label = FeedbackLabel.objects.filter(feedback_aoi__in=aoi_list) + logging.info(label) + + serialized_field = FeedbackLabelFileSerializer(label, many=True) else: - aoi_list = [r.id for r in aois] label = Label.objects.filter(aoi__in=aoi_list) serialized_field = LabelFileSerializer(label, many=True) diff --git a/backend/core/views.py b/backend/core/views.py index 765f1485..66b3fc63 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -244,8 +244,8 @@ class LabelViewSet(viewsets.ModelViewSet): class RawdataApiFeedbackView(APIView): - # authentication_classes = [OsmAuthentication] - # permission_classes = [IsOsmAuthenticated] + authentication_classes = [OsmAuthentication] + permission_classes = [IsOsmAuthenticated] def post(self, request, feedbackaoi_id, *args, **kwargs): """Downloads available osm data as labels within given feedback aoi @@ -408,6 +408,15 @@ def run_task_status(request, run_id: str): class FeedbackView(APIView): + """Applies Associated feedback to Training Published Checkpoint + + Args: + APIView (_type_): _description_ + + Returns: + _type_: _description_ + """ + authentication_classes = [OsmAuthentication] permission_classes = [IsOsmAuthenticated] @@ -421,12 +430,7 @@ def post(self, request, *args, **kwargs): training_id = deserialized_data["training_id"] training_instance = Training.objects.get(id=training_id) - unique_zoom_levels = ( - Feedback.objects.filter(training__id=training_id, validated=True) - .values("zoom_level") - .distinct() - ) - zoom_level = [z["zoom_level"] for z in unique_zoom_levels] + zoom_level = deserialized_data.get("zoom_level", [19, 20]) epochs = deserialized_data.get("epochs", 20) batch_size = deserialized_data.get("batch_size", 8) instance = Training.objects.create( @@ -448,7 +452,7 @@ def post(self, request, *args, **kwargs): zoom_level=instance.zoom_level, source_imagery=instance.source_imagery, feedback=training_id, - freeze_layers=instance.freeze_layers, + freeze_layers=True, # True by default for feedback ) if not instance.source_imagery: instance.source_imagery = instance.model.dataset.source_imagery diff --git a/docker-compose.yml b/docker-compose.yml index 80dfc78e..263647eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - redis - postgres @@ -55,7 +55,7 @@ services: volumes: - ./backend:/app - ${RAMP_HOME}:/RAMP_HOME - # - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE + - ${TRAINING_WORKSPACE}:/TRAINING_WORKSPACE depends_on: - backend-api - redis From 105d7dde53e97ca9361f65e203fd250e0a6fe94b Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 14 Aug 2023 20:54:55 +0545 Subject: [PATCH 07/17] Added feedback aoi gpx and changed url for feedback submit --- backend/core/urls.py | 6 +++++- backend/core/utils.py | 29 +++++++++++++++++++++++++++++ backend/core/views.py | 30 ++++++++++++++---------------- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/backend/core/urls.py b/backend/core/urls.py index d1466c2a..cde78096 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -11,6 +11,7 @@ FeedbackLabelViewset, FeedbackView, FeedbackViewset, + GenerateFeedbackAOIGpxView, GenerateGpxView, LabelViewSet, ModelViewSet, @@ -49,10 +50,13 @@ path("training/status//", run_task_status), path("training/publish//", publish_training), path("prediction/", PredictionView.as_view()), - path("apply/feedback/", FeedbackView.as_view()), + path("feedback/training/submit/", FeedbackView.as_view()), path("status/", APIStatus.as_view()), path("geojson2osm/", geojson2osmconverter, name="geojson2osmconverter"), path("aoi/gpx//", GenerateGpxView.as_view()), + path( + "feedback-aoi/gpx//", GenerateFeedbackAOIGpxView.as_view() + ), path("workspace/", TrainingWorkspaceView.as_view()), path( "workspace/download//", TrainingWorkspaceDownloadView.as_view() diff --git a/backend/core/utils.py b/backend/core/utils.py index 58dcd259..4d6567e9 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -2,12 +2,14 @@ import json import math import os +from datetime import datetime from uuid import uuid4 from xml.dom import ValidationErr from zipfile import ZipFile import requests from django.conf import settings +from gpxpy.gpx import GPX, GPXTrack, GPXTrackSegment, GPXWaypoint from tqdm import tqdm from .models import AOI, FeedbackAOI, FeedbackLabel, Label @@ -243,6 +245,33 @@ def remove_file(path: str) -> None: os.unlink(path) +def gpx_generator(geom_json): + """Generates GPX for give geojson geometry + + Args: + geom_json (_type_): _description_ + + Returns: + xml: gpx + """ + + gpx = GPX() + gpx_track = GPXTrack() + gpx.tracks.append(gpx_track) + gpx_segment = GPXTrackSegment() + gpx_track.segments.append(gpx_segment) + for point in geom_json["coordinates"][0]: + # Append each point as a GPXWaypoint to the GPXTrackSegment + gpx_segment.points.append(GPXWaypoint(point[1], point[0])) + gpx.creator = "fAIr" + gpx_track.name = "Don't Edit this Boundary" + gpx_track.description = "Map inside this boundary and go back to fAIr UI" + gpx.time = datetime.now() + gpx.link = "https://github.com/hotosm/fAIr" + gpx.link_text = "AI Assisted Mapping - fAIr : HOTOSM" + return gpx.to_xml() + + def process_feature(feature, aoi_id, foreign_key_id, feedback=False): """Multi thread process of features""" properties = feature["properties"] diff --git a/backend/core/views.py b/backend/core/views.py index 66b3fc63..c9c4dd16 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -27,7 +27,6 @@ from django_filters.rest_framework import DjangoFilterBackend from drf_yasg.utils import swagger_auto_schema from geojson2osm import geojson2osm -from gpxpy.gpx import GPX, GPXTrack, GPXTrackSegment, GPXWaypoint from hot_fair_utilities import polygonize, predict, vectorize from login.authentication import OsmAuthentication from login.permissions import IsOsmAuthenticated @@ -66,6 +65,7 @@ download_imagery, get_dir_size, get_start_end_download_coords, + gpx_generator, process_rawdata, request_rawdata, ) @@ -624,21 +624,19 @@ def get(self, request, aoi_id: int): # Convert the polygon field to GPX format geom_json = json.loads(aoi.geom.json) # Create a new GPX object - gpx = GPX() - gpx_track = GPXTrack() - gpx.tracks.append(gpx_track) - gpx_segment = GPXTrackSegment() - gpx_track.segments.append(gpx_segment) - for point in geom_json["coordinates"][0]: - # Append each point as a GPXWaypoint to the GPXTrackSegment - gpx_segment.points.append(GPXWaypoint(point[1], point[0])) - gpx.creator = "fAIr Backend" - gpx_track.name = f"AOI of id {aoi_id} , Don't Edit this Boundary" - gpx_track.description = "This is coming from AI Assisted Mapping - fAIr : HOTOSM , Map inside this boundary and go back to fAIr UI" - gpx.time = datetime.now() - gpx.link = "https://github.com/hotosm/fAIr" - gpx.link_text = "AI Assisted Mapping - fAIr : HOTOSM" - return HttpResponse(gpx.to_xml(), content_type="application/xml") + gpx_xml=gpx_generator(geom_json) + return HttpResponse(gpx_xml, content_type="application/xml") + + +class GenerateFeedbackAOIGpxView(APIView): + def get(self, request, feedback_aoi_id: int): + aoi = get_object_or_404(FeedbackAOI, id=feedback_aoi_id) + # Convert the polygon field to GPX format + geom_json = json.loads(aoi.geom.json) + # Create a new GPX object + gpx_xml=gpx_generator(geom_json) + return HttpResponse(gpx_xml, content_type="application/xml") + class TrainingWorkspaceView(APIView): From f83ffedb73c37afb594e2da6f3aa2eae5e1caf2d Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Tue, 15 Aug 2023 11:57:20 +0200 Subject: [PATCH 08/17] feedback till submission and successfull training --- .../AIModels/AIModelEditor/AIModelEditor.js | 2 +- .../components/Layout/Feedback/Feedback.js | 178 +++++++++- .../components/Layout/Feedback/FeedbackAOI.js | 320 +++++++++--------- .../Layout/Feedback/FeedbackTraining.js | 108 ++++++ .../Layout/TrainingDS/DatasetEditor/AOI.js | 2 +- frontend/src/index.css | 7 +- 6 files changed, 447 insertions(+), 170 deletions(-) create mode 100644 frontend/src/components/Layout/Feedback/FeedbackTraining.js diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js index 0aa494db..46591458 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js @@ -30,7 +30,7 @@ const AIModelEditor = (props) => { let { id } = useParams(); const [error, setError] = useState(null); const [epochs, setEpochs] = useState(20); - const [zoomLevel, setZoomLevel] = useState([19, 20]); + const [zoomLevel, setZoomLevel] = useState([19, 20, 21]); const [popupOpen, setPopupOpen] = useState(false); const [sourceImagery, setSourceImagery] = React.useState(null); const [freezeLayers, setFreezeLayers] = useState(false); diff --git a/frontend/src/components/Layout/Feedback/Feedback.js b/frontend/src/components/Layout/Feedback/Feedback.js index 93218d44..ebce61bf 100644 --- a/frontend/src/components/Layout/Feedback/Feedback.js +++ b/frontend/src/components/Layout/Feedback/Feedback.js @@ -1,13 +1,22 @@ import React, { useContext, useEffect, useState } from "react"; -import { useParams } from "react-router-dom"; +import { Link, useNavigate, useParams } from "react-router-dom"; import AuthContext from "../../../Context/AuthContext"; import axios from "../../../axios"; import { useMutation, useQuery } from "react-query"; -import { Alert, AlertTitle, Grid, Typography } from "@mui/material"; +import { + Alert, + AlertTitle, + Grid, + LinearProgress, + Stack, + Typography, +} from "@mui/material"; import { EditControl } from "react-leaflet-draw"; -import { GeoJSON } from "react-leaflet"; +import { GeoJSON, Marker, Popup } from "react-leaflet"; import L from "leaflet"; import "leaflet-draw/dist/leaflet.draw.css"; +import centroid from "@turf/centroid"; +import polygonize from "@turf/polygonize"; import { FeatureGroup, LayersControl, @@ -18,12 +27,20 @@ import { } from "react-leaflet"; import FeedbackAOI from "./FeedbackAOI"; import { approximateGeom, converToGeoPolygon } from "../../../utils"; +import { icon } from "leaflet"; +import FeedackTraining from "./FeedbackTraining"; +const ICON = icon({ + iconUrl: "/hot-marker.png", + iconSize: new L.Point(100, 100), + className: "leaflet-div-icon", +}); const Feedback = (props) => { let { id, trainingId } = useParams(); const { accessToken } = useContext(AuthContext); const [sourceImagery, setSourceImagery] = useState(""); + const [AOIs, setAOIs] = useState(null); const getSourceImagery = async () => { try { const response = await axios.get(`/training/${trainingId}/`); @@ -107,7 +124,7 @@ const Feedback = (props) => { function getFeatureStyle(feature) { return { - color: "red", + color: "green", weight: 3, }; } @@ -140,7 +157,7 @@ const Feedback = (props) => { } }; const { mutate: mutateCreateDB, data: createResult } = useMutation(createDB); - const _onCreate = (e, str) => { + const _onCreate = (e) => { console.log("_onCreate", e); const { layerType, layer } = e; @@ -149,7 +166,6 @@ const Feedback = (props) => { // call the API and add the AOI to DB const newAOI = { id: _leaflet_id, - type: str, latlngs: layer.getLatLngs()[0], area: L.GeometryUtil.geodesicArea(layer.getLatLngs()[0]), }; @@ -160,7 +176,7 @@ const Feedback = (props) => { ) ).slice(1, -2); // console.log("points",points) - const approximated = str === "aoi" ? approximateGeom(points) : points; + const approximated = approximateGeom(points); const polygon = "SRID=4326;POLYGON((" + approximated + "))"; console.log("converToPolygon([layer])", polygon); @@ -171,9 +187,83 @@ const Feedback = (props) => { }); }; const [refresh, setRefresh] = useState(Math.random()); + const getLabels = async (box) => { + try { + console.log(" getLabels for box", box); + + const headers = { + "access-token": accessToken, + }; + const res = await axios.get( + `/feedback-label/?in_bbox=${box._southWest.lng},${box._southWest.lat},${box._northEast.lng},${box._northEast.lat}`, + { headers } + ); + console.log("res from getLabels ", res); + if (res.error) setError(res.error); + else { + // show on the map + let leafletGeoJSON = new L.GeoJSON(res.data); + const newLayers = []; + leafletGeoJSON.eachLayer((layer) => { + const { _leaflet_id, feature } = layer; + // console.log("on get labels layer",layer,layer.getLatLngs(),L.GeometryUtil.geodesicArea(layer.getLatLngs())) + newLayers.push({ + id: _leaflet_id, + aoiId: -1, + feature: feature, + type: "label", + latlngs: layer.getLatLngs()[0], + }); + }); + + return res.data; + } + } catch (e) { + console.log("isError", e); + setError(e); + } finally { + } + }; + const { mutate: mutategetLabels, data: labelsData } = useMutation(getLabels); + + function MyComponent() { + const map = useMapEvents({ + zoomend: (e) => { + const { _animateToZoom } = e.target; + console.log("zoomend", e, _animateToZoom); + setZoom(_animateToZoom); + }, + moveend: (e) => { + const { _animateToZoom, _layers } = e.target; + console.log("moveend", e, e.target.getBounds()); + console.log("zoom is", _animateToZoom); + // console.log("see the map ", map); + + if (_animateToZoom >= 19) { + mutategetLabels(e.target.getBounds()); + } else { + // remote labels layer + } + }, + }); + return null; + } + const navigate = useNavigate(); return ( <> - {!feedbackData && "Loading ..."} + {!feedbackData && ( + + + + + + + + + + + + )} {feedbackData && ( @@ -188,9 +278,14 @@ const Feedback = (props) => { display: "flex", }} zoom={15} + zoomDelta={0.25} + wheelPxPerZoomLevel={Math.round(36 / 0.5)} + zoomSnap={0} + scrollWheelZoom={true} + inertia={true} whenCreated={setMap} > - {/* */} + {oamImagery && ( { style={getFeatureStyle} // onEachFeature={onEachFeature} /> + + {feedbackData && + feedbackData.features.map((f, indx) => { + return ( + + + Feedback + + + ); + })} + + + {zoom >= 19 && ( + + )} { @@ -248,10 +383,30 @@ const Feedback = (props) => { )} + + + { + e.preventDefault(); + navigate("/ai-models/" + id); + }} + > + Model id: {id} + + , Training id: {trainingId} + + + Total feedbacks: {feedbackData && feedbackData.features.length} + + + Zoom: {zoom.toFixed(1)} + + - Mappers feedback is show in red in the map + Mappers feedback is shown in markers on the map @@ -259,11 +414,14 @@ const Feedback = (props) => { {oamImagery && ( )} + )} diff --git a/frontend/src/components/Layout/Feedback/FeedbackAOI.js b/frontend/src/components/Layout/Feedback/FeedbackAOI.js index b287a9d4..9baa3049 100644 --- a/frontend/src/components/Layout/Feedback/FeedbackAOI.js +++ b/frontend/src/components/Layout/Feedback/FeedbackAOI.js @@ -28,6 +28,7 @@ import axios from "../../../axios"; import AOIDetails from "./FeedbackAOIDetails"; import AuthContext from "../../../Context/AuthContext"; import FeedbackAOIDetails from "./FeedbackAOIDetails"; +import area from "@turf/area"; const Demo = styled("div")(({ theme }) => ({ backgroundColor: theme.palette.background.paper, })); @@ -50,8 +51,8 @@ const FeedbackAOI = (props) => { if (res.error) { // setError(res.error.response.statusText); } else { - // console.log("gettraining", res.data); - + console.log(`/feedback-aoi/?training=${props.trainingId}`, res.data); + props.setAOIs(res.data); return res.data; } } catch (e) { @@ -84,16 +85,20 @@ const FeedbackAOI = (props) => { "access-token": accessToken, }; - const res = await axios.post(`/label/osm/fetch/${aoiId}/`, null, { - headers, - }); + const res = await axios.post( + `/label/feedback/osm/fetch/${aoiId}/`, + null, + { + headers, + } + ); if (res.error) { // setMapError(res.error.response.statusText); console.log(res.error.response.statusText); } else { // success full fetch - + props.refresh(); return res.data; } } catch (e) { @@ -130,7 +135,7 @@ const FeedbackAOI = (props) => { const { mutate: mutateDeleteAOI } = useMutation(DeleteAOI); return ( <> - + List of feedback area of Interests{` (${data?.features.length})`} @@ -151,117 +156,117 @@ const FeedbackAOI = (props) => { {data && data.features && data.features.length > 0 && - _DATA.currentData().map((layer) => ( - - - - - - - - Area: {parseInt(layer.area).toLocaleString()} sqm
- - {parseInt(layer.area) < 5000 ? ( - <> - Area seems to be very small for an AOI -
- Make sure it is not a Label - - ) : ( - "" + _DATA.currentData().map((layer) => { + // console.log(layer); + return ( + + + + + + + + Area: {area(layer).toLocaleString()} sqm
+ + {parseInt(layer.area) < 5000 ? ( + <> + Area seems to be very small for an AOI +
+ Make sure it is not a Label + + ) : ( + "" + )} + + {/* add here a container to get the AOI status from DB */} + {layer.id && ( + )} - {/* add here a container to get the AOI status from DB */} - {layer.id && ( - - )} - - } - /> - - {/* + } + /> + + {/* */} - - { - // mutateFetch(layer.aoiId); - // console.log("Open in Editor") - window.open( - `https://rapideditor.org/rapid#background=${ - props.oamImagery - ? "custom:" + props.oamImagery.url + + { + const url = `https://rapideditor.org/rapid#background=${ + props.sourceImagery + ? "custom:" + props.sourceImagery : "Bing" - }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ - layer.aoiId - }`, - "_blank", - "noreferrer" - ); - }} - > - {/* */} - RapiD logo - - - - { - // mutateFetch(layer.aoiId); - // console.log("Open in Editor") - window.open( - `https://www.openstreetmap.org/edit/#background=${ - props.oamImagery - ? "custom:" + props.oamImagery.url + }&datasets=fbRoads,msBuildings&disable_features=boundaries&map=16.00/17.9253/120.4841&gpx=&gpx=https://fair-dev.hotosm.org/api/v1/feedback-aoi/gpx/${ + layer.id + }`; + console.log(url); + window.open(url, "_blank", "noreferrer"); + }} + > + {/* */} + RapiD logo + + + + { + console.log( + "props.sourceImagery", + props.sourceImagery + ); + const url = `https://www.openstreetmap.org/edit/#background=${ + props.sourceImagery + ? "custom:" + props.sourceImagery : "Bing" - }&disable_features=boundaries&gpx=https://fair-dev.hotosm.org/api/v1/aoi/gpx/${ - layer.aoiId - }&map=10.70/18.9226/81.6991`, - "_blank", - "noreferrer" - ); - }} - > - {/* */} - OSM logo - - - - { - mutateFetch(layer.aoiId); - console.log("call raw data API to fetch OSM labels"); - }} - > - - - - {/* + {/* */} + OSM logo + + + + { + mutateFetch(layer.id); + console.log( + "call raw data API to fetch OSM labels" + ); + }} + > + + + + {/* { @@ -270,51 +275,52 @@ const FeedbackAOI = (props) => { }}> */} - - { - const lat = - layer.latlngs.reduce(function ( - accumulator, - curValue - ) { - return accumulator + curValue.lat; - }, - 0) / layer.latlngs.length; - const lng = - layer.latlngs.reduce(function ( - accumulator, - curValue - ) { - return accumulator + curValue.lng; - }, - 0) / layer.latlngs.length; - // [lat, lng] are the centroid of the polygon - props.selectAOIHandler([lat, lng], 17); - }} - > - - - - - { - mutateDeleteAOI(layer.id); - }} - > - - - - - - ))} + + { + const lat = + layer.latlngs.reduce(function ( + accumulator, + curValue + ) { + return accumulator + curValue.lat; + }, + 0) / layer.latlngs.length; + const lng = + layer.latlngs.reduce(function ( + accumulator, + curValue + ) { + return accumulator + curValue.lng; + }, + 0) / layer.latlngs.length; + // [lat, lng] are the centroid of the polygon + props.selectAOIHandler([lat, lng], 17); + }} + > + + + + + { + mutateDeleteAOI(layer.id); + }} + > + + + + + + ); + })} {props.mapLayers && props.mapLayers.length === 0 && ( diff --git a/frontend/src/components/Layout/Feedback/FeedbackTraining.js b/frontend/src/components/Layout/Feedback/FeedbackTraining.js new file mode 100644 index 00000000..be27062a --- /dev/null +++ b/frontend/src/components/Layout/Feedback/FeedbackTraining.js @@ -0,0 +1,108 @@ +import LoadingButton from "@mui/lab/LoadingButton/LoadingButton"; +import { + Checkbox, + FormControl, + FormControlLabel, + FormGroup, + FormLabel, + Grid, + TextField, +} from "@mui/material"; +import React, { useContext, useState } from "react"; +import AuthContext from "../../../Context/AuthContext"; +import axios from "../../../axios"; +import { useMutation } from "react-query"; + +const FeedackTraining = (props) => { + const [epochs, setEpochs] = useState(20); + const [batchSize, setBatchSize] = useState(8); + const [error, setError] = useState(null); + const { accessToken } = useContext(AuthContext); + const publishModel = async () => { + try { + const headers = { + "access-token": accessToken, + }; + const body = { + training_id: props.trainingId, + epochs: epochs, + batch_size: batchSize, + zoom_level: [19, 20, 21], + }; + const res = await axios.post(`/feedback/training/submit/`, body, { + headers, + }); + + if (res.error) { + setError( + res.error.response.statusText + + " / " + + JSON.stringify(res.error.response.data) + ); + return; + } + console.log("training submitted", res.data); + + return res.data; + } catch (e) { + console.log("isError"); + setError(JSON.stringify(e)); + } finally { + } + }; + + const { mutate, isLoading } = useMutation(publishModel); + return ( + + setEpochs(Math.max(0, parseInt(e.target.value)))} + inputProps={{ min: 1, step: 1 }} + fullWidth + margin="normal" + /> + setBatchSize(Math.max(0, parseInt(e.target.value)))} + inputProps={{ min: 1, step: 1 }} + fullWidth + margin="normal" + /> + + Freeze Layers + + setFreezeLayers(e.target.checked)} + name="freeze-layers" + disabled={true} + /> + } + label="Freeze Layers" + /> + + + { + mutate(); + }} + loading={isLoading} + > + Apply Feedback training to Model + + + ); +}; + +export default FeedackTraining; diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js index b5b7b911..f4d32575 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/AOI.js @@ -84,7 +84,7 @@ const AOI = (props) => { return ( <> - + List of Area of Interests{` (${props.mapLayers.length})`} diff --git a/frontend/src/index.css b/frontend/src/index.css index 38b7bab8..1a670d0d 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -147,7 +147,7 @@ code { background-color: transparent !important; } -.leaflet-tile-container img, .leaflet-pane.leaflet-tile-pane img +.leaflet-tile-container img, .leaflet-pane.leaflet-tile-pane img, div.leaflet-pane.leaflet-tile-pane img { padding: 1px; } @@ -164,4 +164,9 @@ code { .margin-left-12 { margin-left: 12px !important; +} + +.leaflet-interactive { + width: 30px !important; + height: 30px !important; } \ No newline at end of file From 93e4728eef8099e547d2f6aa293545170ea63fb2 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Tue, 15 Aug 2023 12:54:04 +0200 Subject: [PATCH 09/17] Show original DS AOIs & sucess message --- .../AIModels/AIModelEditor/AIModelEditor.js | 8 ++- .../components/Layout/Feedback/Feedback.js | 58 ++++++++++++++++++- .../Layout/Feedback/FeedbackTraining.js | 39 ++++++++++++- 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js index 46591458..db274630 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js @@ -80,12 +80,15 @@ const AIModelEditor = (props) => { refetchInterval: 60000, } ); + const [isLoadingFeedbackCount, setIsLoadingFeedbackCount] = useState(true); const getFeedbackCount = async (trainingId) => { try { + setFeedbackCount(0); const response = await axios.get(`/feedback/?training=${trainingId}`); setFeedbackData(response.data); console.log(`/feedback/?training=${trainingId}`, response.data); setFeedbackCount(response.data.features.length); + setIsLoadingFeedbackCount(false); } catch (error) { console.error("Error fetching feedback information:", error); } @@ -366,8 +369,11 @@ const AIModelEditor = (props) => { ); }} disabled={feedbackCount <= 0} + loading={isLoadingFeedbackCount} > - View Feedbacks + {feedbackCount > 0 + ? "View Feedbacks" + : "No feedback for published training"} diff --git a/frontend/src/components/Layout/Feedback/Feedback.js b/frontend/src/components/Layout/Feedback/Feedback.js index ebce61bf..13eb03be 100644 --- a/frontend/src/components/Layout/Feedback/Feedback.js +++ b/frontend/src/components/Layout/Feedback/Feedback.js @@ -76,6 +76,31 @@ const Feedback = (props) => { } finally { } }; + const [originalAOIs, setOriginalAOIs] = useState(null); + const [datasetId, setDatasetId] = useState(null); + const getOriginalAOIs = async () => { + try { + const headers = { + "access-token": accessToken, + }; + const res = await axios.get(`/model/${id}`, null, { + headers, + }); + + if (res.error) { + } else { + const datasetId = res.data.dataset; + setDatasetId(datasetId); + const resAOIs = await axios.get(`/aoi/?dataset=${datasetId}`, null, { + headers, + }); + setOriginalAOIs(resAOIs.data); + } + } catch (e) { + console.log("isError", e); + } finally { + } + }; // const { data: feedbackData, isLoading } = useQuery( // "getFeedback" + trainingId, // getFeedback, @@ -85,7 +110,7 @@ const Feedback = (props) => { // ); useEffect(() => { getFeedback(); - + getOriginalAOIs(); return () => {}; }, []); @@ -249,6 +274,9 @@ const Feedback = (props) => { return null; } const navigate = useNavigate(); + const onEachFeatureOriginalAOIs = (feature, layer) => { + layer.bindPopup("Original dataset AOI"); + }; return ( <> {!feedbackData && ( @@ -337,6 +365,16 @@ const Feedback = (props) => { weight: 4, }} /> + {zoom >= 19 && ( { , Training id: {trainingId} - Total feedbacks: {feedbackData && feedbackData.features.length} + { + e.preventDefault(); + navigate("/training-datasets/" + datasetId); + }} + > + Original dataset id: {datasetId} + + + Total feedbacks: {feedbackData && feedbackData.features.length} + {" "} Zoom: {zoom.toFixed(1)} @@ -421,7 +470,10 @@ const Feedback = (props) => { > )} - + )} diff --git a/frontend/src/components/Layout/Feedback/FeedbackTraining.js b/frontend/src/components/Layout/Feedback/FeedbackTraining.js index be27062a..fd399281 100644 --- a/frontend/src/components/Layout/Feedback/FeedbackTraining.js +++ b/frontend/src/components/Layout/Feedback/FeedbackTraining.js @@ -1,24 +1,29 @@ import LoadingButton from "@mui/lab/LoadingButton/LoadingButton"; import { + Alert, + AlertTitle, Checkbox, FormControl, FormControlLabel, FormGroup, FormLabel, Grid, + Link, TextField, } from "@mui/material"; import React, { useContext, useState } from "react"; import AuthContext from "../../../Context/AuthContext"; import axios from "../../../axios"; import { useMutation } from "react-query"; +import { useNavigate } from "react-router-dom"; const FeedackTraining = (props) => { const [epochs, setEpochs] = useState(20); const [batchSize, setBatchSize] = useState(8); const [error, setError] = useState(null); const { accessToken } = useContext(AuthContext); - const publishModel = async () => { + const navigate = useNavigate(); + const submitFeedbackTraining = async () => { try { const headers = { "access-token": accessToken, @@ -51,7 +56,12 @@ const FeedackTraining = (props) => { } }; - const { mutate, isLoading } = useMutation(publishModel); + const { + mutate, + isLoading, + status, + error: apiError, + } = useMutation(submitFeedbackTraining); return ( { fullWidth margin="normal" /> - + Freeze Layers { > Apply Feedback training to Model + + {status === "success" && ( + + + Training is submitted successfully, go to{" "} + { + e.preventDefault(); + navigate("/ai-models/" + props.modelId); + }} + > + Model id: {props.modelId} + {" "} + for more details + + + )} + {apiError && ( + + {apiError} + + )} ); }; From 9153d9cf413a92ddb138dbaafd5249859dce0782 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Tue, 15 Aug 2023 13:21:59 +0200 Subject: [PATCH 10/17] dataset edirot enhanced layout --- .../TrainingDS/DatasetEditor/DatasetEditor.js | 16 +- .../DatasetEditor/DatasetEditorHeader.js | 138 ++++++------------ .../TrainingDS/DatasetEditor/DatasetMap.js | 33 ++--- 3 files changed, 67 insertions(+), 120 deletions(-) diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js index 7968727c..99fe203d 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js @@ -9,6 +9,7 @@ import MapActions from "./MapActions"; import axios from "../../../../axios"; import { useMutation } from "react-query"; import { useParams } from "react-router-dom"; +import DatasetEditorHeader from "./DatasetEditorHeader"; function DatasetEditor() { const [mapLayers, setMapLayers] = useState([]); @@ -54,7 +55,7 @@ function DatasetEditor() { return () => {}; }, []); - + const [zoom, setZoom] = useState(15); return ( <> {isLoading && "Loading ............"} @@ -77,9 +78,20 @@ function DatasetEditor() { setgeoJSON(null); }} dataset={dataset} + setZoom={setZoom} > - + + + { return ( <> - - - - - Dataset {props.dsId} : {props.dsName} - - + + + Dataset {props.dsId} : {props.dsName} + - - - Zoom: {props.zoom && +props.zoom.toFixed(1)} -
- {"Editing " + props.editMode} - - - - - { - console.log("changed", e); - props.setEditMode(e.target.value); - if (e.target.value === "aoi") { - // console.log("leaflet-bar a",document.querySelectorAll(".leaflet-bar a")) + + Zoom: {props.zoom && +props.zoom.toFixed(1)} + + {/* + + { + console.log("changed", e); + props.setEditMode(e.target.value); + if (e.target.value === "aoi") { + // console.log("leaflet-bar a",document.querySelectorAll(".leaflet-bar a")) - document.querySelectorAll(".leaflet-bar a").forEach((e) => { - e.style.backgroundColor = "rgb(51, 136, 255)"; - console.log("leaflet-bar a", e.style); - }); - } else { - console.log( - "leaflet-bar a", - document.querySelectorAll(".leaflet-bar a") - ); + document.querySelectorAll(".leaflet-bar a").forEach((e) => { + e.style.backgroundColor = "rgb(51, 136, 255)"; + console.log("leaflet-bar a", e.style); + }); + } else { + console.log( + "leaflet-bar a", + document.querySelectorAll(".leaflet-bar a") + ); - document.querySelectorAll(".leaflet-bar a").forEach((e) => { - e.style.backgroundColor = "#D73434"; - console.log("leaflet-bar a", e.style); - }); - } - }} - > - - } - label="AOIs" - labelPlacement="top" - /> - {/* - } - label="Labels" - labelPlacement="top" - /> */} - - + document.querySelectorAll(".leaflet-bar a").forEach((e) => { + e.style.backgroundColor = "#D73434"; + console.log("leaflet-bar a", e.style); + }); + } + }} + > + + + + */} - {/* */} - - - - - - + + ); }; diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js index 11ce4017..01eb0db8 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js @@ -564,6 +564,7 @@ const DatasetMap = (props) => { const { _animateToZoom } = e.target; console.log("zoomend", e, _animateToZoom); setZoom(_animateToZoom); + props.setZoom(_animateToZoom); }, moveend: (e) => { const { _animateToZoom, _layers } = e.target; @@ -597,16 +598,6 @@ const DatasetMap = (props) => { return ( <> - - {mapError && Error: {mapError} } { style={{ height: "800px", width: "100%", - marginTop: "75px", }} zoom={zoom} + zoomDelta={0.25} + wheelPxPerZoomLevel={Math.round(36 / 0.5)} + zoomSnap={0} + scrollWheelZoom={true} + // inertia={true} whenCreated={setMap} > @@ -683,16 +678,12 @@ const DatasetMap = (props) => { { - console.log( - "selectedLayer", - document.querySelector('input[name="selectedLayer"]:checked') - .value - ); - _onCreate( - e, - document.querySelector('input[name="selectedLayer"]:checked') - .value - ); + // console.log( + // "selectedLayer", + // document.querySelector('input[name="selectedLayer"]:checked') + // .value + // ); + _onCreate(e, "aoi"); }} onEdited={_onEdited} onDeleted={_onDeleted} From faa999150449ff1bd80bff78079d4929fce27d91 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Aug 2023 17:40:24 +0545 Subject: [PATCH 11/17] Fixes #153 , Adds validation for feedbackparam --- backend/core/serializers.py | 51 ++++++++++++++++++++++++++++++++----- backend/core/views.py | 10 +++++--- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/backend/core/serializers.py b/backend/core/serializers.py index 9d655aca..a1e6fa43 100644 --- a/backend/core/serializers.py +++ b/backend/core/serializers.py @@ -1,3 +1,4 @@ +from django.conf import settings from login.models import OsmUser from rest_framework import serializers from rest_framework_gis.serializers import ( @@ -164,18 +165,54 @@ class FeedbackParamSerializer(serializers.Serializer): training_id = serializers.IntegerField(required=True) epochs = serializers.IntegerField(required=False) batch_size = serializers.IntegerField(required=False) - zoom_level = serializers.ListField(required=False) + zoom_level = serializers.ListField(child=serializers.IntegerField(), required=False) + + def validate_training_id(self, value): + try: + Training.objects.get(id=value) + except Training.DoesNotExist: + raise serializers.ValidationError("Training doesn't exist") + + return value def validate(self, data): - """ - Check supplied data - """ + training_id = data.get("training_id") + + try: + fd_aois = FeedbackAOI.objects.filter(training=training_id) + except FeedbackAOI.DoesNotExist: + raise serializers.ValidationError( + "No feedback AOI is associated with Training" + ) + + if fd_aois.filter( + label_status=FeedbackAOI.DownloadStatus.NOT_DOWNLOADED + ).exists(): + raise serializers.ValidationError( + "Not all AOIs have their labels downloaded" + ) + + if "epochs" in data and ( + data["epochs"] > settings.EPOCHS_LIMIT or data["epochs"] <= 0 + ): + raise serializers.ValidationError( + f"Epochs should be 1 - {settings.EPOCHS_LIMIT} on this server" + ) + + if "batch_size" in data and ( + data["batch_size"] > settings.BATCH_SIZE_LIMIT or data["batch_size"] <= 0 + ): + raise serializers.ValidationError( + f"Batch size should be 1 - {settings.BATCH_SIZE_LIMIT} on this server" + ) + if "zoom_level" in data: - for i in data["zoom_level"]: - if int(i) < 19 or int(i) > 21: + for zoom in data["zoom_level"]: + if zoom < 19 or zoom > 21: raise serializers.ValidationError( - "Zoom level Supported between 19-21" + "Zoom level must be between 19 and 21" ) + return data diff --git a/backend/core/views.py b/backend/core/views.py index c9c4dd16..8b4a0470 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -425,7 +425,8 @@ class FeedbackView(APIView): ) def post(self, request, *args, **kwargs): res_serializer = FeedbackParamSerializer(data=request.data) - if res_serializer.is_valid(raise_exception=True): + + if res_serializer.is_valid(): deserialized_data = res_serializer.data training_id = deserialized_data["training_id"] training_instance = Training.objects.get(id=training_id) @@ -461,6 +462,8 @@ def post(self, request, *args, **kwargs): print(f"Saved Feedback train model request to queue with id {task.id}") return HttpResponse(status=200) + return Response(res_serializer.errors, status=status.HTTP_400_BAD_REQUEST) + DEFAULT_TILE_SIZE = 256 @@ -624,7 +627,7 @@ def get(self, request, aoi_id: int): # Convert the polygon field to GPX format geom_json = json.loads(aoi.geom.json) # Create a new GPX object - gpx_xml=gpx_generator(geom_json) + gpx_xml = gpx_generator(geom_json) return HttpResponse(gpx_xml, content_type="application/xml") @@ -634,11 +637,10 @@ def get(self, request, feedback_aoi_id: int): # Convert the polygon field to GPX format geom_json = json.loads(aoi.geom.json) # Create a new GPX object - gpx_xml=gpx_generator(geom_json) + gpx_xml = gpx_generator(geom_json) return HttpResponse(gpx_xml, content_type="application/xml") - class TrainingWorkspaceView(APIView): def get(self, request, lookup_dir=None): """List out status of training workspace : size in bytes""" From 35b4a627e5fd1e1eebd5e116f42e1af21cd4fbb3 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Aug 2023 17:59:04 +0545 Subject: [PATCH 12/17] Validation for exisiting trainings --- backend/core/views.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/backend/core/views.py b/backend/core/views.py index 8b4a0470..42ebb504 100644 --- a/backend/core/views.py +++ b/backend/core/views.py @@ -105,6 +105,11 @@ def create(self, validated_data): existing_trainings = Training.objects.filter(model_id=model_id).exclude( status__in=["FINISHED", "FAILED"] ) + if existing_trainings.exists(): + raise ValidationError( + "Another training is already running or submitted for this model." + ) + model = get_object_or_404(Model, id=model_id) if not Label.objects.filter( aoi__in=AOI.objects.filter(dataset=model.dataset) @@ -113,11 +118,6 @@ def create(self, validated_data): "Error: No labels associated with the model, Create AOI & Labels for Dataset" ) - if existing_trainings.exists(): - raise ValidationError( - "Another training is already running or submitted for this model." - ) - epochs = validated_data["epochs"] batch_size = validated_data["batch_size"] @@ -430,10 +430,17 @@ def post(self, request, *args, **kwargs): deserialized_data = res_serializer.data training_id = deserialized_data["training_id"] training_instance = Training.objects.get(id=training_id) + if Training.objects.filter( + model_id=training_instance.model, status__in=["RUNNING", "SUBMITTED"] + ).exists(): + raise ValidationError( + "Another training/feedback is in progress or submitted for this model." + ) zoom_level = deserialized_data.get("zoom_level", [19, 20]) epochs = deserialized_data.get("epochs", 20) batch_size = deserialized_data.get("batch_size", 8) + instance = Training.objects.create( model=training_instance.model, status="SUBMITTED", From a71eb722b1650ed412facfc86717f65406f48508 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Tue, 15 Aug 2023 18:25:17 +0545 Subject: [PATCH 13/17] Add volume for postgres ! --- .gitignore | 1 + docker-compose.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index db7d4292..e54e1552 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ frontend/.pnp.js # testing frontend/coverage +postgres-data/* # production frontend/build diff --git a/docker-compose.yml b/docker-compose.yml index 263647eb..7d949280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ services: - POSTGRES_DB=ai - POSTGRES_USER=postgres - POSTGRES_PASSWORD=admin + volumes: + - ./postgres-data:/var/lib/postgresql/data ports: - "5434:5432" @@ -45,7 +47,7 @@ services: context: ./backend dockerfile: Dockerfile container_name: worker - command: celery -A aiproject worker --loglevel=INFO + command: celery -A aiproject worker --loglevel=INFO --concurrency=1 deploy: resources: reservations: From fcee3f3642561331851b5a77cf9648b1dfd2d228 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Tue, 15 Aug 2023 15:25:01 +0200 Subject: [PATCH 14/17] feedback only on published training --- .../AIModels/AIModelEditor/AIModelEditor.js | 39 ++++++++++--------- .../Layout/Feedback/FeedbackTraining.js | 10 ++++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js index db274630..fa53e6fa 100644 --- a/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js +++ b/frontend/src/components/Layout/AIModels/AIModelEditor/AIModelEditor.js @@ -83,6 +83,7 @@ const AIModelEditor = (props) => { const [isLoadingFeedbackCount, setIsLoadingFeedbackCount] = useState(true); const getFeedbackCount = async (trainingId) => { try { + if (!trainingId) return; setFeedbackCount(0); const response = await axios.get(`/feedback/?training=${trainingId}`); setFeedbackData(response.data); @@ -357,24 +358,26 @@ const AIModelEditor = (props) => { - { - //handleFeedbackClick(data.published_training); - // add logic to view feedbacks here - navigate( - `/ai-models/${data.id}/${data.published_training}/feedback` - ); - }} - disabled={feedbackCount <= 0} - loading={isLoadingFeedbackCount} - > - {feedbackCount > 0 - ? "View Feedbacks" - : "No feedback for published training"} - + {data && data.published_training && ( + { + //handleFeedbackClick(data.published_training); + // add logic to view feedbacks here + navigate( + `/ai-models/${data.id}/${data.published_training}/feedback` + ); + }} + disabled={feedbackCount <= 0} + loading={isLoadingFeedbackCount} + > + {feedbackCount > 0 + ? "View Feedbacks" + : "No feedback for published training"} + + )} {error && ( diff --git a/frontend/src/components/Layout/Feedback/FeedbackTraining.js b/frontend/src/components/Layout/Feedback/FeedbackTraining.js index fd399281..4bc93512 100644 --- a/frontend/src/components/Layout/Feedback/FeedbackTraining.js +++ b/frontend/src/components/Layout/Feedback/FeedbackTraining.js @@ -25,6 +25,7 @@ const FeedackTraining = (props) => { const navigate = useNavigate(); const submitFeedbackTraining = async () => { try { + setError(null); const headers = { "access-token": accessToken, }; @@ -37,7 +38,7 @@ const FeedackTraining = (props) => { const res = await axios.post(`/feedback/training/submit/`, body, { headers, }); - + console.log("/feedback/training/submit", res); if (res.error) { setError( res.error.response.statusText + @@ -112,7 +113,7 @@ const FeedackTraining = (props) => { Apply Feedback training to Model - {status === "success" && ( + {status === "success" && !error && ( Training is submitted successfully, go to{" "} @@ -134,6 +135,11 @@ const FeedackTraining = (props) => { {apiError} )} + {error && ( + + {error} + + )} ); }; From d9a94e66bd09b780d9407f80d9a6b17947cf1834 Mon Sep 17 00:00:00 2001 From: Omran NAJJAR Date: Wed, 16 Aug 2023 13:50:20 +0200 Subject: [PATCH 15/17] load labels on zoom 18 --- frontend/src/components/Layout/Feedback/Feedback.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Layout/Feedback/Feedback.js b/frontend/src/components/Layout/Feedback/Feedback.js index 13eb03be..7e34673b 100644 --- a/frontend/src/components/Layout/Feedback/Feedback.js +++ b/frontend/src/components/Layout/Feedback/Feedback.js @@ -264,7 +264,7 @@ const Feedback = (props) => { console.log("zoom is", _animateToZoom); // console.log("see the map ", map); - if (_animateToZoom >= 19) { + if (_animateToZoom >= 18) { mutategetLabels(e.target.getBounds()); } else { // remote labels layer @@ -375,7 +375,7 @@ const Feedback = (props) => { }} onEachFeature={onEachFeatureOriginalAOIs} /> - {zoom >= 19 && ( + {zoom >= 18 && ( Date: Mon, 21 Aug 2023 12:30:25 +0200 Subject: [PATCH 16/17] support private TMS --- .../TrainingDS/DatasetEditor/DatasetEditor.js | 8 +++ .../TrainingDS/DatasetEditor/DatasetMap.js | 1 + .../DatasetEditor/TileServerList.js | 58 +++++++++++++------ .../TrainingDS/DatasetNew/DatasetNew.js | 16 ++--- 4 files changed, 58 insertions(+), 25 deletions(-) diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js index 99fe203d..6a45c947 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetEditor.js @@ -41,6 +41,14 @@ function DatasetEditor() { if (res.error) setError(res.error.response.statusText); console.log("dataset", res.data); + setOAMImagery({ + center: [0, 0], + name: "Private", + minzoom: 0, + maxzoom: 23, + attribution: res.data.source_imagery, + url: res.data.source_imagery, + }); return res.data; } catch (e) { console.log("isError"); diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js index 01eb0db8..19804fde 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/DatasetMap.js @@ -596,6 +596,7 @@ const DatasetMap = (props) => { return null; } + // console.log("props.oamImagery", props.oamImagery); return ( <> {mapError && Error: {mapError} } diff --git a/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js b/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js index f6889205..2731eeee 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetEditor/TileServerList.js @@ -61,21 +61,34 @@ const TileServerList = (props) => { try { setLoading(true); setInputError(false); - const res = await axios.get(url.replace("/{z}/{x}/{y}", "")); + if (url.includes("openaerial")) { + const res = await axios.get(url.replace("/{z}/{x}/{y}", "")); - if (!res) { - setInputError("Invalid OAM Link"); - return; - } - if (res.error) setInputError(res.error.response.statusText); + if (!res) { + setInputError("Invalid OAM Link"); + return; + } + if (res.error) setInputError(res.error.response.statusText); - props.addImagery(res.data, url); - setImageryDetails(res.data); - props.navigateToCenter(res.data.center); - // console.log("getImageryDetails url",url ) + props.addImagery(res.data, url); + setImageryDetails(res.data); + props.navigateToCenter(res.data.center); + // console.log("getImageryDetails url",url ) - mutateSaveImageryToDataset({ id: props.dataset.id, url }); - return res.data; + mutateSaveImageryToDataset({ id: props.dataset.id, url }); + return res.data; + } else { + // private imagery + mutateSaveImageryToDataset({ id: props.dataset.id, url }); + const obj = { + center: [0, 0], + name: "Private", + minzoom: 15, + maxzoom: 23, + }; + props.addImagery(obj, url); + setImageryDetails(obj); + } } catch (e) { console.log("isError"); // setInputError(e); @@ -96,8 +109,17 @@ const TileServerList = (props) => { console.log("useEffect ", props); // setOAMURL(props.dataset.source_imagery) - if (props.dataset.source_imagery) + if (props.dataset.source_imagery.includes("openaerial")) getImageryDetails(props.dataset.source_imagery); + else { + // load imagery + setImageryDetails({ + center: [0, 0], + name: "Private", + minzoom: 15, + maxzoom: 23, + }); + } return () => {}; }, []); @@ -198,11 +220,11 @@ const TileServerList = (props) => { setInputError(false); let trimmedValue = e.target.value.trim(); let regUrl = /^(https?|chrome):\/\/[^\s$.?#].[^\s]*$/; - let endsWithPng = trimmedValue.endsWith(".png"); - if (endsWithPng) { - trimmedValue = trimmedValue.slice(0, -4); - } - let hasZXY = trimmedValue.endsWith("{z}/{x}/{y}"); + // let endsWithPng = trimmedValue.endsWith(".png"); + // if (endsWithPng) { + // trimmedValue = trimmedValue.slice(0, -4); + // } + let hasZXY = trimmedValue.includes("{z}/{x}/{y}"); let isValid = regUrl.test(trimmedValue) && hasZXY && diff --git a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js index de4aeaf5..91957014 100644 --- a/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js +++ b/frontend/src/components/Layout/TrainingDS/DatasetNew/DatasetNew.js @@ -91,12 +91,14 @@ const DatasetNew = (props) => { fullWidth onChange={(e) => { let trimmedValue = e.target.value.trim(); - let regUrl = /^(https?|chrome):\/\/[^\s$.?#].[^\s]*$/; - let endsWithPng = trimmedValue.endsWith(".png"); - if (endsWithPng) { - trimmedValue = trimmedValue.slice(0, -4); - } - let hasZXY = trimmedValue.endsWith("{z}/{x}/{y}"); + // let regUrl = /^(https?|chrome):\/\/[^\s$.?#].[^\s]*$/; + // let endsWithPng = + // trimmedValue.endsWith(".png") || + // trimmedValue.endsWith(".jpeg"); + // if (endsWithPng) { + // trimmedValue = trimmedValue.slice(0, -4); + // } + let hasZXY = trimmedValue.includes("{z}/{x}/{y}"); let isValid = regUrl.test(trimmedValue) && hasZXY && @@ -123,7 +125,7 @@ const DatasetNew = (props) => { !oamURL || !regUrl.test(oamURL) || DSName.trim() === "" || - !oamURL.endsWith("{z}/{x}/{y}") || + !oamURL.includes("{z}/{x}/{y}") || isLoading } > From 050d20f515e8e156754e0c0f9982340f4b35a74f Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Mon, 21 Aug 2023 18:30:30 +0545 Subject: [PATCH 17/17] Fix bug on downloadin image --- backend/core/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/core/utils.py b/backend/core/utils.py index 4d6567e9..8612258c 100644 --- a/backend/core/utils.py +++ b/backend/core/utils.py @@ -2,6 +2,7 @@ import json import math import os +import re from datetime import datetime from uuid import uuid4 from xml.dom import ValidationErr @@ -111,9 +112,14 @@ def get_start_end_download_coords(bbox_coords, zm_level, tile_size): return start, end +import logging + + def download_image(url, base_path, source_name): response = requests.get(url) image = response.content + url = re.sub(r"\.(png|jpeg)$", "", url) + logging.info(url) url_splitted_list = url.split("/") filename = f"{base_path}/{source_name}-{url_splitted_list[-2]}-{url_splitted_list[-1]}-{url_splitted_list[-3]}.png"