Merge pull request #62 from SkalskiP/develop

1.4.0-alpha relese merge

README.md

@@ -21,7 +21,15 @@ Andrew Ng
 ## Sneak Peek
 
 <p align="center"> 
-    <img width="1000" src=".//examples/alfa-demo.gif" alt="bbox">
+    <img width="1000" src=".//examples/alfa-demo.gif" alt="alfa-demo">
+</p>
+
+## Advanced AI functionalities
+
+[makesense.ai][1] strives to significantly reduce the time we have to spend on labeling photos. To achieve this, we are going to use many different AI models that will be able to give you recommendations as well as automate repetitive and tedious activities. The first step on this journey is to use a [SSD model][8] pretrained on the [COCO dataset][9], which will do some of the work for you in drawing bboxes on photos and - in future versions of the application - will also suggest a label. We also plan to add, among other things, models that classify photos, detect characteristic features of faces, whole faces, and also human pose. The engine that drives our AI functionalities is [TensorFlow.js][10] - JS version of the most popular framework for training neural networks. This choice allows us not only to speed up your work but also to care about the privacy of your data, because unlike with other commercial and open source tools, your photos do not have to be transferred to the server. This time AI comes to your device!
+
+<p align="center"> 
+    <img width="1000" src=".//examples/ai-demo.gif" alt="ai-demo">
 </p>
 
 ## Set Up the Project Locally
@@ -95,17 +103,6 @@ Feel free to file [issues](https://github.com/SkalskiP/make-sense/issues) or [pu
 }
 ```
 
-## Citation
-
-```
-@MISC{make-sense,
-   author = {Piotr Skalski},
-   title = {{Make Sense}},
-   howpublished = "\url{https://github.com/SkalskiP/make-sense/}",
-   year = {2019},
-}
-```
-
 ## License
 
 This project is licensed under the GPL-3.0 License - see the [LICENSE][2] file for details
@@ -119,3 +116,6 @@ Copyright (c) 2019-present, Piotr Skalski
 [5]: https://gitter.im/make-sense-ai/community?utm_source=share-link&utm_medium=link&utm_campaign=share-link
 [6]: https://github.com/SkalskiP/make-sense/wiki/Road-Map
 [7]: https://github.com/SkalskiP/make-sense/wiki/Supported-Output-Formats
+[8]: https://arxiv.org/abs/1512.02325
+[9]: http://cocodataset.org
+[10]: https://www.tensorflow.org/js

package.json

@@ -4,6 +4,8 @@
   "private": true,
   "dependencies": {
     "@material-ui/core": "^4.1.1",
+    "@tensorflow-models/coco-ssd": "^2.0.0",
+    "@tensorflow/tfjs": "^1.2.9",
     "@types/jest": "24.0.14",
     "@types/node": "12.0.8",
     "@types/react": "16.8.20",
@@ -29,7 +31,7 @@
   "scripts": {
     "start": "react-scripts start",
     "build": "react-scripts build",
-    "test": "CI=true react-scripts test --env=jsdom",
+    "test": "CI=true react-scripts test --env=./src/__test__/custom-test-env.js",
     "test:coverage": "npm test -- --coverage",
     "eject": "react-scripts eject"
   },

src/App.scss

@@ -1,8 +1,11 @@
 @import './settings/_Settings';
 
-//.App {
-//  height: 100vh;
-//  width: 100vw;
-//  margin: 0;
-//  padding: 0;
-//}
+.App {
+	--leading-color: #{$secondaryColor};
+	--hue-value: 172deg;
+}
+
+.App.AI {
+	--leading-color: #{$primaryColor};
+	--hue-value: 120deg;
+}

src/App.tsx

@@ -11,13 +11,15 @@ import {ISize} from "./interfaces/ISize";
 import {Settings} from "./settings/Settings";
 import {SizeItUpView} from "./views/SizeItUpView/SizeItUpView";
 import {PlatformModel} from "./staticModels/PlatformModel";
+import classNames from "classnames";
 
 interface IProps {
     projectType: ProjectType;
     windowSize: ISize;
+    AIMode: boolean;
 }
 
-const App: React.FC<IProps> = ({projectType, windowSize}) => {
+const App: React.FC<IProps> = ({projectType, windowSize, AIMode}) => {
     const selectRoute = () => {
         if (!!PlatformModel.mobileDeviceData.manufacturer && !!PlatformModel.mobileDeviceData.os)
             return <MobileMainView/>;
@@ -33,7 +35,7 @@ const App: React.FC<IProps> = ({projectType, windowSize}) => {
     };
 
       return (
-        <div className="App"
+        <div className={classNames("App", {"AI": AIMode})}
             draggable={false}
         >
             {selectRoute()}
@@ -43,8 +45,9 @@ const App: React.FC<IProps> = ({projectType, windowSize}) => {
 };
 
 const mapStateToProps = (state: AppState) => ({
-    projectType: state.editor.projectType,
-    windowSize: state.general.windowSize
+    projectType: state.general.projectData.type,
+    windowSize: state.general.windowSize,
+    AIMode: state.ai.isObjectDetectorLoaded
 });
 
 export default connect(

src/__test__/custom-test-env.js

@@ -0,0 +1,12 @@
+const Environment = require('jest-environment-jsdom');
+
+/**
+ * A custom environment to set the TextEncoder that is required by TensorFlow.js.
+ */
+module.exports = class CustomTestEnvironment extends Environment {
+    async setup() {
+        await super.setup();
+        const { TextEncoder } = require('util');
+        this.global.TextEncoder = TextEncoder;
+    }
+}

src/ai/ObjectDetector.ts

@@ -0,0 +1,39 @@
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import {ObjectDetection} from "@tensorflow-models/coco-ssd";
+import {DetectedObject} from "@tensorflow-models/coco-ssd";
+import {store} from "../index";
+import {updateObjectDetectorStatus} from "../store/ai/actionCreators";
+import {AIActions} from "../logic/actions/AIActions";
+
+export class ObjectDetector {
+    private static model: ObjectDetection;
+
+    public static loadModel(callback?: () => any) {
+        cocoSsd
+            .load()
+            .then((model: ObjectDetection) => {
+                ObjectDetector.model = model;
+                store.dispatch(updateObjectDetectorStatus(true));
+                AIActions.detectRectsForActiveImage();
+                callback && callback();
+            })
+            .catch((error) => {
+                // TODO
+                throw new Error(error);
+            })
+    }
+
+    public static predict(image: HTMLImageElement, callback?: (predictions: DetectedObject[]) => any) {
+        if (!ObjectDetector.model) return;
+
+        ObjectDetector.model
+            .detect(image)
+            .then((predictions: DetectedObject[]) => {
+                callback && callback(predictions)
+            })
+            .catch((error) => {
+                // TODO
+                throw new Error(error);
+            })
+    }
+}

src/data/enums/LabelStatus.ts

@@ -0,0 +1,5 @@
+export enum LabelStatus {
+    ACCEPTED = "ACCEPTED",
+    REJECTED = "REJECTED",
+    UNDECIDED = "UNDECIDED"
+}

src/data/enums/PopupWindowType.ts

@@ -1,6 +1,7 @@
 export enum PopupWindowType {
     LOAD_LABEL_NAMES = "LOAD_LABEL_NAMES",
     LOAD_IMAGES = "LOAD_IMAGES",
+    LOAD_AI_MODEL = "LOAD_AI_MODEL",
     EXPORT_LABELS = "EXPORT_LABELS",
     INSERT_LABEL_NAMES = 'INSERT_LABEL_NAMES',
     EXIT_PROJECT = 'EXIT_PROJECT'

src/data/info/EditorFeatureData.ts

@@ -31,8 +31,8 @@ export const EditorFeatureData: IEditorFeature[] = [
         imageAlt: "file",
     },
     {
-        displayText: "Support basic image operations like crop and resize",
-        imageSrc: "img/crop.png",
-        imageAlt: "crop",
+        displayText: "Use AI to make your work more productive",
+        imageSrc: "img/robot.png",
+        imageAlt: "robot",
     },
 ];

src/logic/actions/AIActions.ts

@@ -0,0 +1,53 @@
+import {DetectedObject} from "@tensorflow-models/coco-ssd";
+import {ImageData, LabelRect} from "../../store/labels/types";
+import {LabelsSelector} from "../../store/selectors/LabelsSelector";
+import uuidv1 from 'uuid/v1';
+import {store} from "../../index";
+import {updateImageDataById} from "../../store/labels/actionCreators";
+import {ObjectDetector} from "../../ai/ObjectDetector";
+import {ImageRepository} from "../imageRepository/ImageRepository";
+import {LabelStatus} from "../../data/enums/LabelStatus";
+
+export class AIActions {
+    public static detectRectsForActiveImage(): void {
+        const activeImageData: ImageData = LabelsSelector.getActiveImageData();
+        AIActions.detectRects(activeImageData.id, ImageRepository.getById(activeImageData.id))
+    }
+
+    public static detectRects(imageId: string, image: HTMLImageElement): void {
+        if (LabelsSelector.getImageDataById(imageId).isVisitedByObjectDetector)
+            return;
+
+        ObjectDetector.predict(image, (predictions: DetectedObject[]) => {
+            AIActions.savePredictions(imageId, predictions);
+        })
+    }
+
+    public static savePredictions(imageId: string, predictions: DetectedObject[]) {
+        const imageData: ImageData = LabelsSelector.getImageDataById(imageId);
+        const predictedLabels: LabelRect[] = AIActions.mapPredictionsToRectLabels(predictions);
+        const nextImageData: ImageData = {
+            ...imageData,
+            labelRects: imageData.labelRects.concat(predictedLabels),
+            isVisitedByObjectDetector: true
+        };
+        store.dispatch(updateImageDataById(imageData.id, nextImageData));
+    }
+
+    public static mapPredictionsToRectLabels(predictions: DetectedObject[]): LabelRect[] {
+        return predictions.map((prediction: DetectedObject) => {
+            return {
+                id: uuidv1(),
+                labelIndex: null,
+                rect: {
+                    x: prediction.bbox[0],
+                    y: prediction.bbox[1],
+                    width: prediction.bbox[2],
+                    height: prediction.bbox[3],
+                },
+                isCreatedByAI: true,
+                status: LabelStatus.UNDECIDED
+            }
+        })
+    }
+}

src/logic/actions/EditorActions.ts

@@ -88,7 +88,7 @@ export class EditorActions {
             viewPortContentSize: CanvasUtil.getSize(EditorModel.canvas),
             activeKeyCombo: ContextManager.getActiveCombo(),
             event: event,
-            zoom: EditorModel.zoom,
+            zoom: GeneralSelector.getZoom(),
             viewPortSize: EditorModel.viewPortSize,
             defaultRenderImageRect: EditorModel.defaultRenderImageRect,
             viewPortContentImageRect: ViewPortActions.calculateViewPortContentImageRect(),