diff --git a/android/app/README.md b/android/app/README.md index ccb007588..4f722caa1 100644 --- a/android/app/README.md +++ b/android/app/README.md @@ -168,6 +168,16 @@ Simple UI for tracking objects of 80 different classes. A short description of t Alt text

+#### Options +- **Dynamic Speed**: reduces the robot speed in "Auto Mode" if it gets closer to the tracked object. + The speed is scaled based on the area of the bouding box (works best in landscape orientation). +- **Model**: choose an object detector based on your phone performance (see below for [benchmarking results](#benchmark)). +- **Object**: pick the object you want to track. The models can detect the 80 COCO [object classes](https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/). +- **Confidence**: confidence threshold to determine if detections are accepted. Increase if you get false detections, decrease if the object of interest it not detected. +- **Device**: use CPU, GPU or NNAPI for inference (more details [here](#device)). +- **Threads**: number of threads to use (only makes a difference when CPU is selected as device). + + ### Point Goal Navigation Note that this fragment requires ARCore and camera permission. If your device does not support ARCore and you continue anyways, the app will crash. In this screen you can specify a goal via a 2D vector with respect to the current position and orientation of the robot. The 2D vector contains the distance to the front and left of the robot in meters. Both values can also be negative and correspond to back and right of the robot in that case. After specifying the goal and pressing `Start` the robot will exectue an AI policy that attempts to reach the goal while avoiding obstacles. diff --git a/android/app/src/main/java/org/openbot/env/SharedPreferencesManager.java b/android/app/src/main/java/org/openbot/env/SharedPreferencesManager.java index 28495ee27..ae4762fb2 100644 --- a/android/app/src/main/java/org/openbot/env/SharedPreferencesManager.java +++ b/android/app/src/main/java/org/openbot/env/SharedPreferencesManager.java @@ -26,7 +26,8 @@ public class SharedPreferencesManager { private static final String OBJECT_TYPE = "OBJECT_TYPE"; private static final String DEFAULT_OBJECT_TYPE = "person"; - + // object tracker switch for speed adjusted by estimated object distance + private static final String OBJECT_NAV_DYNAMIC_SPEED = "OBJECT_NAV_DYNAMICSPEED"; private static final int DEFAULT_DEVICE = Network.Device.CPU.ordinal(); private static final String DEVICE = "DEVICE"; private static final int DEFAULT_NUM_THREAD = 4; @@ -137,6 +138,14 @@ public void setDriveMode(int mode) { preferences.edit().putInt(DRIVE_MODE, mode).apply(); } + public void setDynamicSpeed(boolean isEnabled) { + preferences.edit().putBoolean(OBJECT_NAV_DYNAMIC_SPEED, isEnabled).apply(); + } + + public boolean getDynamicSpeed() { + return preferences.getBoolean(OBJECT_NAV_DYNAMIC_SPEED, false); + } + public void setLogMode(int mode) { preferences.edit().putInt(LOG_MODE, mode).apply(); } diff --git a/android/app/src/main/java/org/openbot/main/SettingsFragment.java b/android/app/src/main/java/org/openbot/main/SettingsFragment.java index 981e512d6..010d4957c 100644 --- a/android/app/src/main/java/org/openbot/main/SettingsFragment.java +++ b/android/app/src/main/java/org/openbot/main/SettingsFragment.java @@ -34,7 +34,6 @@ public class SettingsFragment extends PreferenceFragmentCompat { private SwitchPreferenceCompat storage; private SwitchPreferenceCompat location; private SwitchPreferenceCompat mic; - private final ActivityResultLauncher requestPermissionLauncher = registerForActivityResult( new ActivityResultContracts.RequestMultiplePermissions(), diff --git a/android/app/src/main/java/org/openbot/objectNav/ObjectNavFragment.java b/android/app/src/main/java/org/openbot/objectNav/ObjectNavFragment.java index 3fb67df81..09adee2df 100644 --- a/android/app/src/main/java/org/openbot/objectNav/ObjectNavFragment.java +++ b/android/app/src/main/java/org/openbot/objectNav/ObjectNavFragment.java @@ -206,6 +206,12 @@ public void onNothingSelected(AdapterView parent) {} Enums.SpeedMode.getByID(preferencesManager.getSpeedMode())))); binding.autoSwitch.setOnClickListener(v -> setNetworkEnabled(binding.autoSwitch.isChecked())); + binding.dynamicSpeed.setChecked(preferencesManager.getDynamicSpeed()); + binding.dynamicSpeed.setOnClickListener( + v -> { + preferencesManager.setDynamicSpeed(binding.dynamicSpeed.isChecked()); + tracker.setDynamicSpeed(preferencesManager.getDynamicSpeed()); + }); } private void updateCropImageInfo() { @@ -223,6 +229,7 @@ private void updateCropImageInfo() { borderedText.setTypeface(Typeface.MONOSPACE); tracker = new MultiBoxTracker(requireContext()); + tracker.setDynamicSpeed(preferencesManager.getDynamicSpeed()); Timber.i("Camera orientation relative to screen canvas: %d", sensorOrientation); @@ -378,6 +385,7 @@ private void setNetworkEnabledWithAudio(boolean b) { private void setNetworkEnabled(boolean b) { binding.autoSwitch.setChecked(b); + binding.controllerContainer.controlMode.setEnabled(!b); binding.controllerContainer.driveMode.setEnabled(!b); binding.controllerContainer.speedInfo.setEnabled(!b); diff --git a/android/app/src/main/java/org/openbot/original/CameraActivity.java b/android/app/src/main/java/org/openbot/original/CameraActivity.java index 74a3e29e0..0f4e49e72 100755 --- a/android/app/src/main/java/org/openbot/original/CameraActivity.java +++ b/android/app/src/main/java/org/openbot/original/CameraActivity.java @@ -181,7 +181,7 @@ public abstract class CameraActivity extends AppCompatActivity private Intent intentSensorService; private ServerCommunication serverCommunication; - private SharedPreferencesManager preferencesManager; + protected SharedPreferencesManager preferencesManager; protected final GameController gameController = new GameController(driveMode); private PhoneController phoneController; protected final ControllerHandler controllerHandler = new ControllerHandler(); @@ -452,6 +452,10 @@ public void onReceive(Context context, Intent intent) { vehicle.requestVehicleConfig(); } + /** + * Initalize bottom sheet views/fields with shared preference values (@see + * SharedPreferenceManager) + */ @SuppressLint("SetTextI18n") private void setInitialValues() { cameraSwitchCompat.setChecked(preferencesManager.getCameraSwitch()); diff --git a/android/app/src/main/java/org/openbot/original/DefaultActivity.java b/android/app/src/main/java/org/openbot/original/DefaultActivity.java index f13d6429e..17ceed98a 100755 --- a/android/app/src/main/java/org/openbot/original/DefaultActivity.java +++ b/android/app/src/main/java/org/openbot/original/DefaultActivity.java @@ -100,6 +100,7 @@ public void onPreviewSizeChosen(final Size size, final int rotation) { borderedText.setTypeface(Typeface.MONOSPACE); tracker = new MultiBoxTracker(this); + tracker.setDynamicSpeed(preferencesManager.getDynamicSpeed()); previewWidth = size.getWidth(); previewHeight = size.getHeight(); diff --git a/android/app/src/main/java/org/openbot/tracking/MultiBoxTracker.java b/android/app/src/main/java/org/openbot/tracking/MultiBoxTracker.java index 0886e2ce4..699b98cf9 100755 --- a/android/app/src/main/java/org/openbot/tracking/MultiBoxTracker.java +++ b/android/app/src/main/java/org/openbot/tracking/MultiBoxTracker.java @@ -73,6 +73,7 @@ public class MultiBoxTracker { private int sensorOrientation; private float leftControl; private float rightControl; + private boolean useDynamicSpeed = false; public MultiBoxTracker(final Context context) { for (final int color : COLORS) { @@ -143,21 +144,30 @@ private void updateFrameToCanvasMatrix(int canvasHeight, int canvasWidth) { false); } + /** + * Determine the robot controls/steering from the position of the tracked object/person on screen. + * The follow speed is adjusted based on the area of the bounding box of the tracked object. + * Assumption: large object box --> close to object --> slow down + * + * @return the adjusted speed control for left and right wheels in the range -1.0 ... 1.0 + */ public synchronized Control updateTarget() { if (!trackedObjects.isEmpty()) { - // Pick person with highest probability + // Pick detection with highest probability final RectF trackedPos = new RectF(trackedObjects.get(0).location); final boolean rotated = sensorOrientation % 180 == 90; float imgWidth = (float) (rotated ? frameHeight : frameWidth); + // calculate track box area for distance estimate + float boxArea = trackedPos.height() * trackedPos.width(); float centerX = (rotated ? trackedPos.centerY() : trackedPos.centerX()); // Make sure object center is in frame centerX = Math.max(0.0f, Math.min(centerX, imgWidth)); // Scale relative position along x-axis between -1 and 1 float x_pos_norm = 1.0f - 2.0f * centerX / imgWidth; - // Scale to control signal and account for rotation + // Scale for steering signal and account for rotation, float x_pos_scaled = rotated ? -x_pos_norm * 1.0f : x_pos_norm * 1.0f; //// Scale by "exponential" function: y = x / sqrt(1-x^2) - // Math.max (Math.min(x_pos_norm / Math.sqrt(1 - x_pos_norm * x_pos_norm),2),-2) * 255.0f; + // Math.max (Math.min(x_pos_norm / Math.sqrt(1 - x_pos_norm * x_pos_norm),2),-2); if (x_pos_scaled < 0) { leftControl = 1.0f; @@ -166,10 +176,26 @@ public synchronized Control updateTarget() { leftControl = 1.0f - x_pos_scaled; rightControl = 1.0f; } + + // adjust speed depending on size of detected object bounding box + if (useDynamicSpeed) { + float scaleFactor = 1.0f - boxArea / (frameWidth * frameHeight); + scaleFactor = scaleFactor > 0.75f ? 1.0f : scaleFactor; // tracked object far, full speed + // apply scale factor if tracked object is not too near, otherwise stop + if (scaleFactor > 0.25f) { + leftControl *= scaleFactor; + rightControl *= scaleFactor; + } else { + leftControl = 0.0f; + rightControl = 0.0f; + } + } + } else { leftControl = 0.0f; rightControl = 0.0f; } + return new Control( (0 > sensorOrientation) ? rightControl : leftControl, (0 > sensorOrientation) ? leftControl : rightControl); @@ -262,6 +288,15 @@ private void processResults(final List results) { } } + /** + * Set use of dynamic speed on or off (used in updateTarget()) + * + * @param isEnabled + */ + public void setDynamicSpeed(boolean isEnabled) { + useDynamicSpeed = isEnabled; + } + private static class TrackedRecognition { RectF location; float detectionConfidence; diff --git a/android/app/src/main/res/layout-land/fragment_object_nav.xml b/android/app/src/main/res/layout-land/fragment_object_nav.xml index 62383b13c..90072987f 100644 --- a/android/app/src/main/res/layout-land/fragment_object_nav.xml +++ b/android/app/src/main/res/layout-land/fragment_object_nav.xml @@ -44,17 +44,57 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toTopOf="parent" /> - + + + + + app:layout_constraintTop_toBottomOf="@+id/view"> + + + + + + - - - - - - diff --git a/android/app/src/main/res/layout/fragment_object_nav.xml b/android/app/src/main/res/layout/fragment_object_nav.xml index e7bb8ddbb..7d8ee42ee 100644 --- a/android/app/src/main/res/layout/fragment_object_nav.xml +++ b/android/app/src/main/res/layout/fragment_object_nav.xml @@ -32,18 +32,34 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toTopOf="parent" /> - + app:layout_constraintTop_toBottomOf="@+id/view"> + + + + + diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 5f321b9de..a75b47c29 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -92,6 +92,8 @@ Tracking lost. No initial AR Core pose. AR Core session paused. + Auto Mode + Dynamic Speed Bumpers N/A diff --git a/docs/images/screen_object_tracking_1.jpg b/docs/images/screen_object_tracking_1.jpg index fe9eb5269..9dc738f53 100644 Binary files a/docs/images/screen_object_tracking_1.jpg and b/docs/images/screen_object_tracking_1.jpg differ diff --git a/docs/images/screen_object_tracking_2.jpg b/docs/images/screen_object_tracking_2.jpg index 6937bd447..ff3cb992a 100644 Binary files a/docs/images/screen_object_tracking_2.jpg and b/docs/images/screen_object_tracking_2.jpg differ