From c38e6fdfbf7943d06f94e008a9d34d41417d1a72 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 15:18:23 +0530 Subject: [PATCH 01/17] Update Gradle distribution URL to version 7.2 --- .../android/gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties b/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties index 9b7cb4af38a..ef426240827 100644 --- a/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties +++ b/lite/examples/object_detection/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip networkTimeout=10000 retries=0 retryBackOffMs=500 From 74e7bf96ea90902061a1a28e0e0a26d915888b2b Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 15:19:55 +0530 Subject: [PATCH 02/17] Add GitHub Actions workflow for Android build --- .github/workflows/android-build.yml | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/android-build.yml diff --git a/.github/workflows/android-build.yml b/.github/workflows/android-build.yml new file mode 100644 index 00000000000..5724405aa09 --- /dev/null +++ b/.github/workflows/android-build.yml @@ -0,0 +1,41 @@ + +name: Build Android Object Detection Demo + +on: + workflow_dispatch: + push: + paths: + - "lite/examples/object_detection/android/**" + - ".github/workflows/android-build.yml" + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + + - name: Build debug APK + working-directory: lite/examples/object_detection/android + run: | + chmod +x gradlew + ./gradlew assembleDebug --stacktrace + + - name: Upload APK + if: success() + uses: actions/upload-artifact@v4 + with: + name: blind-assist-debug-apk + path: lite/examples/object_detection/android/app/build/outputs/apk/debug/*.apk + if-no-files-found: error + retention-days: 7 From 4091dc72eb8cf6747092a44b4a80ccd9278cf17a Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 15:47:25 +0530 Subject: [PATCH 03/17] Update AndroidManifest.xml with improved comments --- .../android/app/src/main/AndroidManifest.xml | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml index a2e00b2b63b..4b1c17cf23a 100644 --- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml +++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml @@ -1,18 +1,19 @@ + - - + + - + + + + + + + + - + - - + + From 7997e60643aa6f9803751a0005d191d500cde943 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 15:48:05 +0530 Subject: [PATCH 04/17] Implement TextToSpeech for audio warnings Added TextToSpeech functionality to provide audio warnings. --- .../examples/objectdetection/MainActivity.kt | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt index f65924e40e1..54fd042cf59 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/MainActivity.kt @@ -1,44 +1,58 @@ + /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Licensed under the Apache License, Version 2.0. */ package org.tensorflow.lite.examples.objectdetection import android.os.Build import android.os.Bundle +import android.speech.tts.TextToSpeech import androidx.appcompat.app.AppCompatActivity import org.tensorflow.lite.examples.objectdetection.databinding.ActivityMainBinding +import java.util.Locale -/** - * Main entry point into our app. This app follows the single-activity pattern, and all - * functionality is implemented in the form of fragments. - */ class MainActivity : AppCompatActivity() { private lateinit var activityMainBinding: ActivityMainBinding + private var textToSpeech: TextToSpeech? = null + private var ttsReady = false override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) activityMainBinding = ActivityMainBinding.inflate(layoutInflater) setContentView(activityMainBinding.root) + + textToSpeech = TextToSpeech(this) { status -> + if (status == TextToSpeech.SUCCESS) { + val result = textToSpeech?.setLanguage(Locale.US) + ttsReady = result != TextToSpeech.LANG_MISSING_DATA && + result != TextToSpeech.LANG_NOT_SUPPORTED + } + } + } + + private fun speakTestWarning() { + if (ttsReady) { + textToSpeech?.speak( + "Person ahead", + TextToSpeech.QUEUE_FLUSH, + null, + "test_warning" + ) + } + } + + override fun onDestroy() { + textToSpeech?.stop() + textToSpeech?.shutdown() + textToSpeech = null + super.onDestroy() } override fun onBackPressed() { if (Build.VERSION.SDK_INT == Build.VERSION_CODES.Q) { - // Workaround for Android Q memory leak issue in IRequestFinishCallback$Stub. - // (https://issuetracker.google.com/issues/139738913) finishAfterTransition() } else { super.onBackPressed() From 552037748914941d3b781227e982eb46600fff67 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:01:37 +0530 Subject: [PATCH 05/17] Clean up AndroidManifest.xml comments and license Removed unnecessary comments and simplified license information. --- .../android/app/src/main/AndroidManifest.xml | 22 ++----------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml index 4b1c17cf23a..0fffd9a1da4 100644 --- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml +++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml @@ -2,18 +2,8 @@ - - - - @@ -58,19 +44,15 @@ android:configChanges="orientation|screenLayout|screenSize|smallestScreenSize" tools:targetApi="O"> - - - - From 8cb790f606ae6b529dcb91810fb862788724c480 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:07:10 +0530 Subject: [PATCH 06/17] Update AndroidManifest.xml with camera and internet permissions Added required permissions and features for camera and internet access. --- .../object_detection/android/app/src/main/AndroidManifest.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml index 0fffd9a1da4..6c09487114f 100644 --- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml +++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml @@ -13,12 +13,16 @@ + + + + From e25d7216bec432f08e45d9d36e4545c943402f40 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:10:38 +0530 Subject: [PATCH 07/17] Clean up comments in AndroidManifest.xml Removed unnecessary comments from AndroidManifest.xml. --- .../object_detection/android/app/src/main/AndroidManifest.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml index 6c09487114f..629a354f1d6 100644 --- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml +++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml @@ -1,5 +1,4 @@ - - - From d367156920e6be31d0dd05623f142d9df092cbb3 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:45:55 +0530 Subject: [PATCH 08/17] Integrate Text-to-Speech for object detection announcements Added Text-to-Speech functionality to announce detected objects and improved code structure. --- .../fragments/CameraFragment.kt | 374 +++++++++++------- 1 file changed, 226 insertions(+), 148 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index afc13d8f3fc..a7a512db4d7 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -1,24 +1,17 @@ + /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Licensed under the Apache License, Version 2.0. */ + package org.tensorflow.lite.examples.objectdetection.fragments import android.annotation.SuppressLint import android.content.res.Configuration import android.graphics.Bitmap import android.os.Bundle +import android.speech.tts.TextToSpeech import android.util.Log import android.view.LayoutInflater import android.view.View @@ -37,6 +30,7 @@ import androidx.core.content.ContextCompat import androidx.fragment.app.Fragment import androidx.navigation.Navigation import java.util.LinkedList +import java.util.Locale import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import org.tensorflow.lite.examples.objectdetection.ObjectDetectorHelper @@ -55,38 +49,57 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { private lateinit var objectDetectorHelper: ObjectDetectorHelper private lateinit var bitmapBuffer: Bitmap + private var preview: Preview? = null private var imageAnalyzer: ImageAnalysis? = null private var camera: Camera? = null private var cameraProvider: ProcessCameraProvider? = null - /** Blocking camera operations are performed using this executor */ - private lateinit var cameraExecutor: ExecutorService + // Text-to-Speech + private var textToSpeech: TextToSpeech? = null + private var ttsReady = false + private var lastSpokenLabel = "" + private var lastSpokenTime = 0L + + private val cameraExecutor: ExecutorService by lazy { + Executors.newSingleThreadExecutor() + } override fun onResume() { super.onResume() - // Make sure that all permissions are still present, since the - // user could have removed them while the app was in paused state. + if (!PermissionsFragment.hasPermissions(requireContext())) { - Navigation.findNavController(requireActivity(), R.id.fragment_container) - .navigate(CameraFragmentDirections.actionCameraToPermissions()) + Navigation.findNavController( + requireActivity(), + R.id.fragment_container + ).navigate(CameraFragmentDirections.actionCameraToPermissions()) } } override fun onDestroyView() { + textToSpeech?.stop() + textToSpeech?.shutdown() + textToSpeech = null + ttsReady = false + + cameraProvider?.unbindAll() + cameraProvider = null + + if (::cameraExecutor.isInitialized) { + cameraExecutor.shutdown() + } + _fragmentCameraBinding = null super.onDestroyView() - - // Shut down our background executor - cameraExecutor.shutdown() } override fun onCreateView( - inflater: LayoutInflater, - container: ViewGroup?, - savedInstanceState: Bundle? + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? ): View { - _fragmentCameraBinding = FragmentCameraBinding.inflate(inflater, container, false) + _fragmentCameraBinding = + FragmentCameraBinding.inflate(inflater, container, false) return fragmentCameraBinding.root } @@ -95,232 +108,297 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) + initializeTextToSpeech() + objectDetectorHelper = ObjectDetectorHelper( context = requireContext(), - objectDetectorListener = this) + objectDetectorListener = this + ) - // Initialize our background executor - cameraExecutor = Executors.newSingleThreadExecutor() + // Initialize the camera background executor. + cameraExecutor - // Wait for the views to be properly laid out fragmentCameraBinding.viewFinder.post { - // Set up the camera and its use cases setUpCamera() } - // Attach listeners to UI control widgets initBottomSheetControls() } - private fun initBottomSheetControls() { - // When clicked, lower detection score threshold floor - fragmentCameraBinding.bottomSheetLayout.thresholdMinus.setOnClickListener { - if (objectDetectorHelper.threshold >= 0.1) { - objectDetectorHelper.threshold -= 0.1f - updateControlsUi() + private fun initializeTextToSpeech() { + ttsReady = false + + textToSpeech = TextToSpeech(requireContext()) { status -> + if (status == TextToSpeech.SUCCESS) { + val result = textToSpeech?.setLanguage(Locale.US) + + ttsReady = result != null && + result != TextToSpeech.LANG_MISSING_DATA && + result != TextToSpeech.LANG_NOT_SUPPORTED + + Log.d(TAG, "Text-to-Speech ready: $ttsReady") + } else { + Log.e(TAG, "Text-to-Speech initialization failed") } } + } - // When clicked, raise detection score threshold floor - fragmentCameraBinding.bottomSheetLayout.thresholdPlus.setOnClickListener { - if (objectDetectorHelper.threshold <= 0.8) { - objectDetectorHelper.threshold += 0.1f - updateControlsUi() + private fun announceDetection(results: MutableList?) { + if (!ttsReady || results.isNullOrEmpty()) return + + val bestCategory = results + .asSequence() + .mapNotNull { detection -> + detection.categories.maxByOrNull { it.score } } + .filter { it.score >= 0.60f } + .maxByOrNull { it.score } + ?: return + + val label = bestCategory.label + if (label.isBlank()) return + + val now = System.currentTimeMillis() + + // Announce a new label immediately. Repeat the same label + // no more frequently than once every three seconds. + if (label == lastSpokenLabel && + now - lastSpokenTime < 3000L + ) { + return } - // When clicked, reduce the number of objects that can be detected at a time - fragmentCameraBinding.bottomSheetLayout.maxResultsMinus.setOnClickListener { - if (objectDetectorHelper.maxResults > 1) { - objectDetectorHelper.maxResults-- - updateControlsUi() - } + val accepted = textToSpeech?.speak( + "$label ahead", + TextToSpeech.QUEUE_FLUSH, + null, + "object_warning" + ) + + if (accepted == TextToSpeech.SUCCESS) { + lastSpokenLabel = label + lastSpokenTime = now } + } - // When clicked, increase the number of objects that can be detected at a time - fragmentCameraBinding.bottomSheetLayout.maxResultsPlus.setOnClickListener { - if (objectDetectorHelper.maxResults < 5) { - objectDetectorHelper.maxResults++ - updateControlsUi() + private fun initBottomSheetControls() { + fragmentCameraBinding.bottomSheetLayout.thresholdMinus + .setOnClickListener { + if (objectDetectorHelper.threshold >= 0.1f) { + objectDetectorHelper.threshold -= 0.1f + updateControlsUi() + } } - } - // When clicked, decrease the number of threads used for detection - fragmentCameraBinding.bottomSheetLayout.threadsMinus.setOnClickListener { - if (objectDetectorHelper.numThreads > 1) { - objectDetectorHelper.numThreads-- - updateControlsUi() + fragmentCameraBinding.bottomSheetLayout.thresholdPlus + .setOnClickListener { + if (objectDetectorHelper.threshold <= 0.8f) { + objectDetectorHelper.threshold += 0.1f + updateControlsUi() + } } - } - // When clicked, increase the number of threads used for detection - fragmentCameraBinding.bottomSheetLayout.threadsPlus.setOnClickListener { - if (objectDetectorHelper.numThreads < 4) { - objectDetectorHelper.numThreads++ - updateControlsUi() + fragmentCameraBinding.bottomSheetLayout.maxResultsMinus + .setOnClickListener { + if (objectDetectorHelper.maxResults > 1) { + objectDetectorHelper.maxResults-- + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.maxResultsPlus + .setOnClickListener { + if (objectDetectorHelper.maxResults < 5) { + objectDetectorHelper.maxResults++ + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.threadsMinus + .setOnClickListener { + if (objectDetectorHelper.numThreads > 1) { + objectDetectorHelper.numThreads-- + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.threadsPlus + .setOnClickListener { + if (objectDetectorHelper.numThreads < 4) { + objectDetectorHelper.numThreads++ + updateControlsUi() + } } - } - // When clicked, change the underlying hardware used for inference. Current options are CPU - // GPU, and NNAPI - fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection(0, false) - fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.onItemSelectedListener = + fragmentCameraBinding.bottomSheetLayout.spinnerDelegate + .setSelection(0, false) + + fragmentCameraBinding.bottomSheetLayout.spinnerDelegate + .onItemSelectedListener = object : AdapterView.OnItemSelectedListener { - override fun onItemSelected(p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long) { - objectDetectorHelper.currentDelegate = p2 + override fun onItemSelected( + parent: AdapterView<*>?, + view: View?, + position: Int, + id: Long + ) { + objectDetectorHelper.currentDelegate = position updateControlsUi() } - override fun onNothingSelected(p0: AdapterView<*>?) { - /* no op */ + override fun onNothingSelected(parent: AdapterView<*>?) { + // No action required. } } - // When clicked, change the underlying model used for object detection - fragmentCameraBinding.bottomSheetLayout.spinnerModel.setSelection(0, false) - fragmentCameraBinding.bottomSheetLayout.spinnerModel.onItemSelectedListener = + fragmentCameraBinding.bottomSheetLayout.spinnerModel + .setSelection(0, false) + + fragmentCameraBinding.bottomSheetLayout.spinnerModel + .onItemSelectedListener = object : AdapterView.OnItemSelectedListener { - override fun onItemSelected(p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long) { - objectDetectorHelper.currentModel = p2 + override fun onItemSelected( + parent: AdapterView<*>?, + view: View?, + position: Int, + id: Long + ) { + objectDetectorHelper.currentModel = position updateControlsUi() } - override fun onNothingSelected(p0: AdapterView<*>?) { - /* no op */ + override fun onNothingSelected(parent: AdapterView<*>?) { + // No action required. } } } - // Update the values displayed in the bottom sheet. Reset detector. private fun updateControlsUi() { fragmentCameraBinding.bottomSheetLayout.maxResultsValue.text = objectDetectorHelper.maxResults.toString() + fragmentCameraBinding.bottomSheetLayout.thresholdValue.text = String.format("%.2f", objectDetectorHelper.threshold) + fragmentCameraBinding.bottomSheetLayout.threadsValue.text = objectDetectorHelper.numThreads.toString() - // Needs to be cleared instead of reinitialized because the GPU - // delegate needs to be initialized on the thread using it when applicable objectDetectorHelper.clearObjectDetector() fragmentCameraBinding.overlay.clear() } - // Initialize CameraX, and prepare to bind the camera use cases private fun setUpCamera() { - val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext()) + val cameraProviderFuture = + ProcessCameraProvider.getInstance(requireContext()) + cameraProviderFuture.addListener( { - // CameraProvider cameraProvider = cameraProviderFuture.get() - - // Build and bind the camera use cases bindCameraUseCases() }, ContextCompat.getMainExecutor(requireContext()) ) } - // Declare and bind preview, capture and analysis use cases @SuppressLint("UnsafeOptInUsageError") private fun bindCameraUseCases() { - - // CameraProvider - val cameraProvider = - cameraProvider ?: throw IllegalStateException("Camera initialization failed.") - - // CameraSelector - makes assumption that we're only using the back camera - val cameraSelector = - CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build() - - // Preview. Only using the 4:3 ratio because this is the closest to our models - preview = - Preview.Builder() - .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) - .build() - - // ImageAnalysis. Using RGBA 8888 to match how our models work - imageAnalyzer = - ImageAnalysis.Builder() - .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) - .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) - .build() - // The analyzer can then be assigned to the instance - .also { - it.setAnalyzer(cameraExecutor) { image -> - if (!::bitmapBuffer.isInitialized) { - // The image rotation and RGB image buffer are initialized only once - // the analyzer has started running - bitmapBuffer = Bitmap.createBitmap( - image.width, - image.height, - Bitmap.Config.ARGB_8888 - ) - } - - detectObjects(image) + val provider = cameraProvider + ?: throw IllegalStateException("Camera initialization failed.") + + val cameraSelector = CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build() + + preview = Preview.Builder() + .setTargetAspectRatio(AspectRatio.RATIO_4_3) + .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) + .build() + + imageAnalyzer = ImageAnalysis.Builder() + .setTargetAspectRatio(AspectRatio.RATIO_4_3) + .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) + .setBackpressureStrategy( + ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST + ) + .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) + .build() + .also { analyzer -> + analyzer.setAnalyzer(cameraExecutor) { image -> + if (!::bitmapBuffer.isInitialized) { + bitmapBuffer = Bitmap.createBitmap( + image.width, + image.height, + Bitmap.Config.ARGB_8888 + ) } + + detectObjects(image) } + } - // Must unbind the use-cases before rebinding them - cameraProvider.unbindAll() + provider.unbindAll() try { - // A variable number of use-cases can be passed here - - // camera provides access to CameraControl & CameraInfo - camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalyzer) + camera = provider.bindToLifecycle( + this, + cameraSelector, + preview, + imageAnalyzer + ) - // Attach the viewfinder's surface provider to preview use case - preview?.setSurfaceProvider(fragmentCameraBinding.viewFinder.surfaceProvider) + preview?.setSurfaceProvider( + fragmentCameraBinding.viewFinder.surfaceProvider + ) } catch (exc: Exception) { Log.e(TAG, "Use case binding failed", exc) } } private fun detectObjects(image: ImageProxy) { - // Copy out RGB bits to the shared bitmap buffer - image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) } + image.use { + bitmapBuffer.copyPixelsFromBuffer(it.planes[0].buffer) + } val imageRotation = image.imageInfo.rotationDegrees - // Pass Bitmap and rotation to the object detector helper for processing and detection objectDetectorHelper.detect(bitmapBuffer, imageRotation) } override fun onConfigurationChanged(newConfig: Configuration) { super.onConfigurationChanged(newConfig) - imageAnalyzer?.targetRotation = fragmentCameraBinding.viewFinder.display.rotation + imageAnalyzer?.targetRotation = + fragmentCameraBinding.viewFinder.display.rotation } - // Update UI after objects have been detected. Extracts original image height/width - // to scale and place bounding boxes properly through OverlayView override fun onResults( - results: MutableList?, - inferenceTime: Long, - imageHeight: Int, - imageWidth: Int + results: MutableList?, + inferenceTime: Long, + imageHeight: Int, + imageWidth: Int ) { activity?.runOnUiThread { fragmentCameraBinding.bottomSheetLayout.inferenceTimeVal.text = - String.format("%d ms", inferenceTime) + String.format("%d ms", inferenceTime) - // Pass necessary information to OverlayView for drawing on the canvas fragmentCameraBinding.overlay.setResults( results ?: LinkedList(), imageHeight, imageWidth ) - // Force a redraw fragmentCameraBinding.overlay.invalidate() + + // Announce the best qualifying detection. + announceDetection(results) } } override fun onError(error: String) { activity?.runOnUiThread { - Toast.makeText(requireContext(), error, Toast.LENGTH_SHORT).show() + Toast.makeText( + requireContext(), + error, + Toast.LENGTH_SHORT + ).show() } } } From 229419d1f4c809710a11022c3f5327dea7f5280a Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:46:56 +0530 Subject: [PATCH 09/17] Update AndroidManifest.xml with permissions and queries Added permissions for camera and internet access, and allowed discovery of Text-to-Speech engines. --- .../object_detection/android/app/src/main/AndroidManifest.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml index 629a354f1d6..51fbbe13108 100644 --- a/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml +++ b/lite/examples/object_detection/android/app/src/main/AndroidManifest.xml @@ -16,9 +16,11 @@ android:name="android.hardware.camera" android:required="true" /> + + From 8096d6430b54e69dd90c562747265a4161f2725c Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 16:51:32 +0530 Subject: [PATCH 10/17] Refactor cameraExecutor initialization and usage --- .../fragments/CameraFragment.kt | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index a7a512db4d7..4100c390636 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -49,6 +49,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { private lateinit var objectDetectorHelper: ObjectDetectorHelper private lateinit var bitmapBuffer: Bitmap + private lateinit var cameraExecutor: ExecutorService private var preview: Preview? = null private var imageAnalyzer: ImageAnalysis? = null @@ -61,10 +62,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { private var lastSpokenLabel = "" private var lastSpokenTime = 0L - private val cameraExecutor: ExecutorService by lazy { - Executors.newSingleThreadExecutor() - } - override fun onResume() { super.onResume() @@ -116,7 +113,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { ) // Initialize the camera background executor. - cameraExecutor + cameraExecutor = Executors.newSingleThreadExecutor() fragmentCameraBinding.viewFinder.post { setUpCamera() @@ -160,9 +157,10 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { val now = System.currentTimeMillis() - // Announce a new label immediately. Repeat the same label - // no more frequently than once every three seconds. - if (label == lastSpokenLabel && + // Announce a new label immediately. + // Repeat the same label at most once every three seconds. + if ( + label == lastSpokenLabel && now - lastSpokenTime < 3000L ) { return @@ -324,15 +322,21 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { .build() .also { analyzer -> analyzer.setAnalyzer(cameraExecutor) { image -> - if (!::bitmapBuffer.isInitialized) { - bitmapBuffer = Bitmap.createBitmap( - image.width, - image.height, - Bitmap.Config.ARGB_8888 - ) + try { + if (!::bitmapBuffer.isInitialized) { + bitmapBuffer = Bitmap.createBitmap( + image.width, + image.height, + Bitmap.Config.ARGB_8888 + ) + } + + detectObjects(image) + } catch (exc: Exception) { + Log.e(TAG, "Object detection failed", exc) + } finally { + image.close() } - - detectObjects(image) } } @@ -355,9 +359,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } private fun detectObjects(image: ImageProxy) { - image.use { - bitmapBuffer.copyPixelsFromBuffer(it.planes[0].buffer) - } + bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) val imageRotation = image.imageInfo.rotationDegrees objectDetectorHelper.detect(bitmapBuffer, imageRotation) @@ -376,29 +378,33 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { imageWidth: Int ) { activity?.runOnUiThread { - fragmentCameraBinding.bottomSheetLayout.inferenceTimeVal.text = + val binding = _fragmentCameraBinding ?: return@runOnUiThread + + binding.bottomSheetLayout.inferenceTimeVal.text = String.format("%d ms", inferenceTime) - fragmentCameraBinding.overlay.setResults( + binding.overlay.setResults( results ?: LinkedList(), imageHeight, imageWidth ) - fragmentCameraBinding.overlay.invalidate() + binding.overlay.invalidate() - // Announce the best qualifying detection. + // Speak the best detection above the confidence threshold. announceDetection(results) } } override fun onError(error: String) { activity?.runOnUiThread { - Toast.makeText( - requireContext(), - error, - Toast.LENGTH_SHORT - ).show() + context?.let { safeContext -> + Toast.makeText( + safeContext, + error, + Toast.LENGTH_SHORT + ).show() + } } } } From aec93724cfc032ae80481d8a69dd2c04b631b06e Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 17:54:57 +0530 Subject: [PATCH 11/17] Refactor ObjectDetectorHelper with default values --- .../objectdetection/ObjectDetectorHelper.kt | 237 ++++++++++++------ 1 file changed, 155 insertions(+), 82 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt index 7fb4f0ef282..563c008fa4a 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/ObjectDetectorHelper.kt @@ -1,3 +1,4 @@ + /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * @@ -13,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.tensorflow.lite.examples.objectdetection import android.content.Context @@ -28,126 +30,197 @@ import org.tensorflow.lite.task.vision.detector.Detection import org.tensorflow.lite.task.vision.detector.ObjectDetector class ObjectDetectorHelper( - var threshold: Float = 0.5f, - var numThreads: Int = 2, - var maxResults: Int = 3, - var currentDelegate: Int = 0, - var currentModel: Int = 0, - val context: Context, - val objectDetectorListener: DetectorListener? + var threshold: Float = DEFAULT_THRESHOLD, + var numThreads: Int = DEFAULT_NUM_THREADS, + var maxResults: Int = DEFAULT_MAX_RESULTS, + var currentDelegate: Int = DELEGATE_CPU, + var currentModel: Int = MODEL_EFFICIENTDETV0, + val context: Context, + val objectDetectorListener: DetectorListener? ) { - // For this example this needs to be a var so it can be reset on changes. If the ObjectDetector - // will not change, a lazy val would be preferable. + companion object { + private const val TAG = "ObjectDetectorHelper" + + // Detection configuration + const val DEFAULT_THRESHOLD = 0.45f + const val DEFAULT_NUM_THREADS = 4 + const val DEFAULT_MAX_RESULTS = 5 + + // Hardware delegates + const val DELEGATE_CPU = 0 + const val DELEGATE_GPU = 1 + const val DELEGATE_NNAPI = 2 + + // Model identifiers + const val MODEL_MOBILENETV1 = 0 + const val MODEL_EFFICIENTDETV0 = 1 + const val MODEL_EFFICIENTDETV1 = 2 + const val MODEL_EFFICIENTDETV2 = 3 + } + private var objectDetector: ObjectDetector? = null init { setupObjectDetector() } + /** + * Releases the current detector reference. + * Call setupObjectDetector() afterwards to initialize it again. + */ fun clearObjectDetector() { objectDetector = null } - // Initialize the object detector using current settings on the - // thread that is using it. CPU and NNAPI delegates can be used with detectors - // that are created on the main thread and used on a background thread, but - // the GPU delegate needs to be used on the thread that initialized the detector + /** + * Creates the detector using the selected model, confidence threshold, + * result limit, thread count and hardware delegate. + */ + @Synchronized fun setupObjectDetector() { - // Create the base options for the detector using specifies max results and score threshold - val optionsBuilder = - ObjectDetector.ObjectDetectorOptions.builder() - .setScoreThreshold(threshold) - .setMaxResults(maxResults) - - // Set general detection options, including number of used threads - val baseOptionsBuilder = BaseOptions.builder().setNumThreads(numThreads) - - // Use the specified hardware for running the model. Default to CPU - when (currentDelegate) { - DELEGATE_CPU -> { - // Default - } - DELEGATE_GPU -> { - if (CompatibilityList().isDelegateSupportedOnThisDevice) { - baseOptionsBuilder.useGpu() - } else { - objectDetectorListener?.onError("GPU is not supported on this device") + // Release the previous detector reference before changing configuration. + objectDetector = null + + try { + val optionsBuilder = + ObjectDetector.ObjectDetectorOptions.builder() + .setScoreThreshold(threshold.coerceIn(0.0f, 1.0f)) + .setMaxResults(maxResults.coerceAtLeast(1)) + + val baseOptionsBuilder = + BaseOptions.builder() + .setNumThreads(numThreads.coerceAtLeast(1)) + + when (currentDelegate) { + DELEGATE_CPU -> { + // CPU is the default and most compatible option. + } + + DELEGATE_GPU -> { + val compatibilityList = CompatibilityList() + + if (compatibilityList.isDelegateSupportedOnThisDevice) { + baseOptionsBuilder.useGpu() + } else { + objectDetectorListener?.onError( + "GPU is not supported on this device. Using CPU instead." + ) + Log.w(TAG, "GPU delegate unsupported; using CPU.") + } + } + + DELEGATE_NNAPI -> { + baseOptionsBuilder.useNnapi() + } + + else -> { + Log.w(TAG, "Unknown delegate $currentDelegate; using CPU.") } } - DELEGATE_NNAPI -> { - baseOptionsBuilder.useNnapi() - } - } - optionsBuilder.setBaseOptions(baseOptionsBuilder.build()) + optionsBuilder.setBaseOptions(baseOptionsBuilder.build()) - val modelName = - when (currentModel) { + val modelName = when (currentModel) { MODEL_MOBILENETV1 -> "mobilenetv1.tflite" MODEL_EFFICIENTDETV0 -> "efficientdet-lite0.tflite" MODEL_EFFICIENTDETV1 -> "efficientdet-lite1.tflite" MODEL_EFFICIENTDETV2 -> "efficientdet-lite2.tflite" - else -> "mobilenetv1.tflite" + else -> { + Log.w(TAG, "Unknown model $currentModel; using MobileNet V1.") + "mobilenetv1.tflite" + } } - try { objectDetector = - ObjectDetector.createFromFileAndOptions(context, modelName, optionsBuilder.build()) - } catch (e: IllegalStateException) { + ObjectDetector.createFromFileAndOptions( + context, + modelName, + optionsBuilder.build() + ) + + Log.i( + TAG, + "Detector initialized: model=$modelName, " + + "threshold=$threshold, threads=$numThreads, " + + "maxResults=$maxResults, delegate=$currentDelegate" + ) + } catch (e: Exception) { + objectDetector = null + + Log.e( + TAG, + "Failed to initialize object detector.", + e + ) + objectDetectorListener?.onError( - "Object detector failed to initialize. See error logs for details" + "Could not initialize the object detector. " + + "Check the model file and device configuration." ) - Log.e("Test", "TFLite failed to load model with error: " + e.message) } } + /** + * Runs object detection on a bitmap. + * + * imageRotation should be the camera rotation in degrees, normally + * 0, 90, 180 or 270, as supplied by the existing camera pipeline. + */ fun detect(image: Bitmap, imageRotation: Int) { - if (objectDetector == null) { + val detector = objectDetector ?: run { setupObjectDetector() + objectDetector } - // Inference time is the difference between the system time at the start and finish of the - // process - var inferenceTime = SystemClock.uptimeMillis() - - // Create preprocessor for the image. - // See https://www.tensorflow.org/lite/inference_with_metadata/ - // lite_support#imageprocessor_architecture - val imageProcessor = - ImageProcessor.Builder() - .add(Rot90Op(-imageRotation / 90)) - .build() - - // Preprocess the image and convert it into a TensorImage for detection. - val tensorImage = imageProcessor.process(TensorImage.fromBitmap(image)) - - val results = objectDetector?.detect(tensorImage) - inferenceTime = SystemClock.uptimeMillis() - inferenceTime - objectDetectorListener?.onResults( - results, - inferenceTime, - tensorImage.height, - tensorImage.width) + if (detector == null) { + Log.e(TAG, "Detection skipped because the detector is unavailable.") + return + } + + val startTime = SystemClock.elapsedRealtime() + + try { + val rotation = ((imageRotation % 360) + 360) % 360 + + val imageProcessor = + ImageProcessor.Builder() + .add(Rot90Op(-rotation / 90)) + .build() + + val tensorImage = + imageProcessor.process( + TensorImage.fromBitmap(image) + ) + + val results = detector.detect(tensorImage) + + val inferenceTime = + SystemClock.elapsedRealtime() - startTime + + objectDetectorListener?.onResults( + results, + inferenceTime, + tensorImage.height, + tensorImage.width + ) + } catch (e: Exception) { + Log.e(TAG, "Object detection failed.", e) + + objectDetectorListener?.onError( + "Object detection failed. Please try again." + ) + } } interface DetectorListener { fun onError(error: String) + fun onResults( - results: MutableList?, - inferenceTime: Long, - imageHeight: Int, - imageWidth: Int + results: MutableList?, + inferenceTime: Long, + imageHeight: Int, + imageWidth: Int ) } - - companion object { - const val DELEGATE_CPU = 0 - const val DELEGATE_GPU = 1 - const val DELEGATE_NNAPI = 2 - const val MODEL_MOBILENETV1 = 0 - const val MODEL_EFFICIENTDETV0 = 1 - const val MODEL_EFFICIENTDETV1 = 2 - const val MODEL_EFFICIENTDETV2 = 3 - } } From 773111c5dec14fc2a19161f01b1e55586bb95857 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 17:56:17 +0530 Subject: [PATCH 12/17] Refactor CameraFragment for clarity and efficiency Refactor CameraFragment to improve readability and maintainability by organizing constants, enhancing text-to-speech functionality, and ensuring proper resource management. --- .../fragments/CameraFragment.kt | 234 ++++++++++++++---- 1 file changed, 183 insertions(+), 51 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index 4100c390636..4c92acca997 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -2,7 +2,17 @@ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.tensorflow.lite.examples.objectdetection.fragments @@ -11,6 +21,7 @@ import android.annotation.SuppressLint import android.content.res.Configuration import android.graphics.Bitmap import android.os.Bundle +import android.os.SystemClock import android.speech.tts.TextToSpeech import android.util.Log import android.view.LayoutInflater @@ -40,7 +51,21 @@ import org.tensorflow.lite.task.vision.detector.Detection class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { - private val TAG = "ObjectDetection" + companion object { + private const val TAG = "ObjectDetection" + + // A label must appear in this many consecutive analyzed frames. + private const val REQUIRED_STABLE_FRAMES = 3 + + // Minimum time between any two announcements. + private const val MIN_ANNOUNCEMENT_GAP_MS = 2000L + + // Minimum delay before announcing the same label again. + private const val SAME_LABEL_REPEAT_DELAY_MS = 6000L + + // Minimum confidence required before a detection can be spoken. + private const val SPEECH_CONFIDENCE_THRESHOLD = 0.60f + } private var _fragmentCameraBinding: FragmentCameraBinding? = null @@ -59,8 +84,14 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { // Text-to-Speech private var textToSpeech: TextToSpeech? = null private var ttsReady = false + + // Stable detection tracking + private var candidateLabel = "" + private var candidateCount = 0 + + // Announcement tracking private var lastSpokenLabel = "" - private var lastSpokenTime = 0L + private var lastAnnouncementTime = 0L override fun onResume() { super.onResume() @@ -69,19 +100,29 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { Navigation.findNavController( requireActivity(), R.id.fragment_container - ).navigate(CameraFragmentDirections.actionCameraToPermissions()) + ).navigate( + CameraFragmentDirections.actionCameraToPermissions() + ) } } override fun onDestroyView() { + // Stop speech and release the TTS engine. textToSpeech?.stop() textToSpeech?.shutdown() textToSpeech = null ttsReady = false + // Stop camera analysis. + imageAnalyzer?.clearAnalyzer() cameraProvider?.unbindAll() + + imageAnalyzer = null + preview = null + camera = null cameraProvider = null + // Release the background executor. if (::cameraExecutor.isInitialized) { cameraExecutor.shutdown() } @@ -105,6 +146,8 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { override fun onViewCreated(view: View, savedInstanceState: Bundle?) { super.onViewCreated(view, savedInstanceState) + cameraExecutor = Executors.newSingleThreadExecutor() + initializeTextToSpeech() objectDetectorHelper = ObjectDetectorHelper( @@ -112,14 +155,14 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { objectDetectorListener = this ) - // Initialize the camera background executor. - cameraExecutor = Executors.newSingleThreadExecutor() - fragmentCameraBinding.viewFinder.post { - setUpCamera() + if (_fragmentCameraBinding != null) { + setUpCamera() + } } initBottomSheetControls() + updateControlsUi() } private fun initializeTextToSpeech() { @@ -135,63 +178,104 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { Log.d(TAG, "Text-to-Speech ready: $ttsReady") } else { + ttsReady = false Log.e(TAG, "Text-to-Speech initialization failed") } } } + /** + * Selects the highest-confidence label for the current frame. + * The label must remain stable for several frames before it is spoken. + */ private fun announceDetection(results: MutableList?) { - if (!ttsReady || results.isNullOrEmpty()) return + if (!ttsReady || textToSpeech == null) return val bestCategory = results - .asSequence() - .mapNotNull { detection -> + ?.asSequence() + ?.mapNotNull { detection -> detection.categories.maxByOrNull { it.score } } - .filter { it.score >= 0.60f } - .maxByOrNull { it.score } - ?: return + ?.filter { category -> + category.score >= SPEECH_CONFIDENCE_THRESHOLD && + category.label.isNotBlank() + } + ?.maxByOrNull { it.score } + + // No sufficiently confident detection in this frame. + if (bestCategory == null) { + candidateLabel = "" + candidateCount = 0 + return + } val label = bestCategory.label - if (label.isBlank()) return - val now = System.currentTimeMillis() + // Require the same label in consecutive frames. + if (label == candidateLabel) { + candidateCount++ + } else { + candidateLabel = label + candidateCount = 1 + } + + if (candidateCount < REQUIRED_STABLE_FRAMES) { + return + } + + val now = SystemClock.elapsedRealtime() + + // Avoid speaking too frequently, even when different objects appear. + if (now - lastAnnouncementTime < MIN_ANNOUNCEMENT_GAP_MS) { + return + } - // Announce a new label immediately. - // Repeat the same label at most once every three seconds. + // Avoid repeating the same label too often. if ( label == lastSpokenLabel && - now - lastSpokenTime < 3000L + now - lastAnnouncementTime < SAME_LABEL_REPEAT_DELAY_MS ) { return } - val accepted = textToSpeech?.speak( - "$label ahead", + val result = textToSpeech?.speak( + "$label detected", TextToSpeech.QUEUE_FLUSH, null, "object_warning" ) - if (accepted == TextToSpeech.SUCCESS) { + if (result == TextToSpeech.SUCCESS) { lastSpokenLabel = label - lastSpokenTime = now + lastAnnouncementTime = now + + // Require fresh stable frames before the next announcement. + candidateLabel = "" + candidateCount = 0 + + Log.d(TAG, "Announced object: $label") } } private fun initBottomSheetControls() { fragmentCameraBinding.bottomSheetLayout.thresholdMinus .setOnClickListener { - if (objectDetectorHelper.threshold >= 0.1f) { - objectDetectorHelper.threshold -= 0.1f + if (objectDetectorHelper.threshold > 0.1f) { + objectDetectorHelper.threshold = + (objectDetectorHelper.threshold - 0.1f) + .coerceAtLeast(0.1f) + updateControlsUi() } } fragmentCameraBinding.bottomSheetLayout.thresholdPlus .setOnClickListener { - if (objectDetectorHelper.threshold <= 0.8f) { - objectDetectorHelper.threshold += 0.1f + if (objectDetectorHelper.threshold < 0.9f) { + objectDetectorHelper.threshold = + (objectDetectorHelper.threshold + 0.1f) + .coerceAtMost(0.9f) + updateControlsUi() } } @@ -234,14 +318,17 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { fragmentCameraBinding.bottomSheetLayout.spinnerDelegate .onItemSelectedListener = object : AdapterView.OnItemSelectedListener { + override fun onItemSelected( parent: AdapterView<*>?, view: View?, position: Int, id: Long ) { - objectDetectorHelper.currentDelegate = position - updateControlsUi() + if (objectDetectorHelper.currentDelegate != position) { + objectDetectorHelper.currentDelegate = position + updateControlsUi() + } } override fun onNothingSelected(parent: AdapterView<*>?) { @@ -255,14 +342,17 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { fragmentCameraBinding.bottomSheetLayout.spinnerModel .onItemSelectedListener = object : AdapterView.OnItemSelectedListener { + override fun onItemSelected( parent: AdapterView<*>?, view: View?, position: Int, id: Long ) { - objectDetectorHelper.currentModel = position - updateControlsUi() + if (objectDetectorHelper.currentModel != position) { + objectDetectorHelper.currentModel = position + updateControlsUi() + } } override fun onNothingSelected(parent: AdapterView<*>?) { @@ -272,36 +362,60 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } private fun updateControlsUi() { - fragmentCameraBinding.bottomSheetLayout.maxResultsValue.text = + val binding = _fragmentCameraBinding ?: return + + binding.bottomSheetLayout.maxResultsValue.text = objectDetectorHelper.maxResults.toString() - fragmentCameraBinding.bottomSheetLayout.thresholdValue.text = - String.format("%.2f", objectDetectorHelper.threshold) + binding.bottomSheetLayout.thresholdValue.text = + String.format( + Locale.US, + "%.2f", + objectDetectorHelper.threshold + ) - fragmentCameraBinding.bottomSheetLayout.threadsValue.text = + binding.bottomSheetLayout.threadsValue.text = objectDetectorHelper.numThreads.toString() + // Reset speech stability when detector settings change. + candidateLabel = "" + candidateCount = 0 + objectDetectorHelper.clearObjectDetector() - fragmentCameraBinding.overlay.clear() + binding.overlay.clear() } private fun setUpCamera() { + val context = context ?: return + val cameraProviderFuture = - ProcessCameraProvider.getInstance(requireContext()) + ProcessCameraProvider.getInstance(context) cameraProviderFuture.addListener( { - cameraProvider = cameraProviderFuture.get() - bindCameraUseCases() + if (_fragmentCameraBinding == null) return@addListener + + try { + cameraProvider = cameraProviderFuture.get() + bindCameraUseCases() + } catch (exc: Exception) { + Log.e(TAG, "Camera initialization failed", exc) + onError("Unable to initialize the camera.") + } }, - ContextCompat.getMainExecutor(requireContext()) + ContextCompat.getMainExecutor(context) ) } @SuppressLint("UnsafeOptInUsageError") private fun bindCameraUseCases() { val provider = cameraProvider - ?: throw IllegalStateException("Camera initialization failed.") + ?: run { + onError("Camera provider is unavailable.") + return + } + + val binding = _fragmentCameraBinding ?: return val cameraSelector = CameraSelector.Builder() .requireLensFacing(CameraSelector.LENS_FACING_BACK) @@ -309,12 +423,12 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { preview = Preview.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) + .setTargetRotation(binding.viewFinder.display.rotation) .build() imageAnalyzer = ImageAnalysis.Builder() .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation) + .setTargetRotation(binding.viewFinder.display.rotation) .setBackpressureStrategy( ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST ) @@ -323,7 +437,15 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { .also { analyzer -> analyzer.setAnalyzer(cameraExecutor) { image -> try { - if (!::bitmapBuffer.isInitialized) { + if (_fragmentCameraBinding == null) { + return@setAnalyzer + } + + if ( + !::bitmapBuffer.isInitialized || + bitmapBuffer.width != image.width || + bitmapBuffer.height != image.height + ) { bitmapBuffer = Bitmap.createBitmap( image.width, image.height, @@ -351,24 +473,34 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { ) preview?.setSurfaceProvider( - fragmentCameraBinding.viewFinder.surfaceProvider + binding.viewFinder.surfaceProvider ) } catch (exc: Exception) { Log.e(TAG, "Use case binding failed", exc) + onError("Unable to start the camera preview.") } } private fun detectObjects(image: ImageProxy) { + if (!::bitmapBuffer.isInitialized) return + bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) val imageRotation = image.imageInfo.rotationDegrees - objectDetectorHelper.detect(bitmapBuffer, imageRotation) + + objectDetectorHelper.detect( + bitmapBuffer, + imageRotation + ) } override fun onConfigurationChanged(newConfig: Configuration) { super.onConfigurationChanged(newConfig) - imageAnalyzer?.targetRotation = - fragmentCameraBinding.viewFinder.display.rotation + + _fragmentCameraBinding?.let { binding -> + imageAnalyzer?.targetRotation = + binding.viewFinder.display.rotation + } } override fun onResults( @@ -378,10 +510,11 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { imageWidth: Int ) { activity?.runOnUiThread { - val binding = _fragmentCameraBinding ?: return@runOnUiThread + val binding = _fragmentCameraBinding + ?: return@runOnUiThread binding.bottomSheetLayout.inferenceTimeVal.text = - String.format("%d ms", inferenceTime) + String.format(Locale.US, "%d ms", inferenceTime) binding.overlay.setResults( results ?: LinkedList(), @@ -391,7 +524,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { binding.overlay.invalidate() - // Speak the best detection above the confidence threshold. announceDetection(results) } } From e77f3129a61492423b2c3be846e328e4145608fc Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 17:57:22 +0530 Subject: [PATCH 13/17] Refactor OverlayView for improved readability and performance --- .../examples/objectdetection/OverlayView.kt | 211 +++++++++++++----- 1 file changed, 156 insertions(+), 55 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt index 605afff3c60..b660fe84150 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/OverlayView.kt @@ -1,3 +1,4 @@ + /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * @@ -26,96 +27,196 @@ import android.util.AttributeSet import android.view.View import androidx.core.content.ContextCompat import java.util.LinkedList +import java.util.Locale import kotlin.math.max +import kotlin.math.min import org.tensorflow.lite.task.vision.detector.Detection -class OverlayView(context: Context?, attrs: AttributeSet?) : View(context, attrs) { +class OverlayView( + context: Context?, + attrs: AttributeSet? +) : View(context, attrs) { + + private var results: List = LinkedList() - private var results: List = LinkedList() - private var boxPaint = Paint() - private var textBackgroundPaint = Paint() - private var textPaint = Paint() + private val boxPaint = Paint(Paint.ANTI_ALIAS_FLAG) + private val textBackgroundPaint = Paint(Paint.ANTI_ALIAS_FLAG) + private val textPaint = Paint(Paint.ANTI_ALIAS_FLAG) private var scaleFactor: Float = 1f + private val bounds = Rect() - private var bounds = Rect() + // Layout settings + private val boxStrokeWidth = 5f + private val textSizeSp = 14f + private val textPadding = 8f + private val cornerRadius = 8f init { initPaints() } + private fun initPaints() { + val density = resources.displayMetrics.density + val scaledTextSize = textSizeSp * density + + boxPaint.apply { + color = ContextCompat.getColor( + context ?: return@apply, + R.color.bounding_box_color + ) + style = Paint.Style.STROKE + strokeWidth = boxStrokeWidth * density + } + + textBackgroundPaint.apply { + color = Color.BLACK + alpha = 220 + style = Paint.Style.FILL + } + + textPaint.apply { + color = Color.WHITE + style = Paint.Style.FILL + textSize = scaledTextSize + typeface = android.graphics.Typeface.create( + android.graphics.Typeface.DEFAULT, + android.graphics.Typeface.BOLD + ) + } + } + + /** + * Clears all displayed detections. + */ fun clear() { - textPaint.reset() - textBackgroundPaint.reset() - boxPaint.reset() + results = emptyList() invalidate() - initPaints() } - private fun initPaints() { - textBackgroundPaint.color = Color.BLACK - textBackgroundPaint.style = Paint.Style.FILL - textBackgroundPaint.textSize = 50f + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) - textPaint.color = Color.WHITE - textPaint.style = Paint.Style.FILL - textPaint.textSize = 50f + if (results.isEmpty() || width <= 0 || height <= 0) { + return + } - boxPaint.color = ContextCompat.getColor(context!!, R.color.bounding_box_color) - boxPaint.strokeWidth = 8F - boxPaint.style = Paint.Style.STROKE - } + for (result in results) { + val category = result.categories.maxByOrNull { it.score } + ?: continue - override fun draw(canvas: Canvas) { - super.draw(canvas) + val label = category.label.ifBlank { "Unknown object" } + val confidence = (category.score * 100f) + .coerceIn(0f, 100f) - for (result in results) { val boundingBox = result.boundingBox - val top = boundingBox.top * scaleFactor - val bottom = boundingBox.bottom * scaleFactor val left = boundingBox.left * scaleFactor + val top = boundingBox.top * scaleFactor val right = boundingBox.right * scaleFactor + val bottom = boundingBox.bottom * scaleFactor - // Draw bounding box around detected objects - val drawableRect = RectF(left, top, right, bottom) - canvas.drawRect(drawableRect, boxPaint) - - // Create text to display alongside detected objects - val drawableText = - result.categories[0].label + " " + - String.format("%.2f", result.categories[0].score) - - // Draw rect behind display text - textBackgroundPaint.getTextBounds(drawableText, 0, drawableText.length, bounds) - val textWidth = bounds.width() - val textHeight = bounds.height() - canvas.drawRect( + val drawableRect = RectF( left, top, - left + textWidth + Companion.BOUNDING_RECT_TEXT_PADDING, - top + textHeight + Companion.BOUNDING_RECT_TEXT_PADDING, + right, + bottom + ) + + // Draw the bounding box. + canvas.drawRoundRect( + drawableRect, + cornerRadius, + cornerRadius, + boxPaint + ) + + // Display label and confidence as a percentage. + val drawableText = String.format( + Locale.US, + "%s %.0f%%", + label, + confidence + ) + + textPaint.getTextBounds( + drawableText, + 0, + drawableText.length, + bounds + ) + + val textWidth = textPaint.measureText(drawableText) + val textHeight = bounds.height().toFloat() + + val backgroundWidth = textWidth + textPadding * 2 + val backgroundHeight = textHeight + textPadding * 2 + + // Prefer placing the label above the bounding box. + // If there is insufficient room, place it inside the box. + val backgroundTop = if (top >= backgroundHeight) { + top - backgroundHeight + } else { + top + } + + val backgroundLeft = left.coerceAtLeast(0f) + + val backgroundRect = RectF( + backgroundLeft, + backgroundTop, + min(backgroundLeft + backgroundWidth, width.toFloat()), + min(backgroundTop + backgroundHeight, height.toFloat()) + ) + + canvas.drawRoundRect( + backgroundRect, + cornerRadius, + cornerRadius, textBackgroundPaint ) - // Draw text for detected object - canvas.drawText(drawableText, left, top + bounds.height(), textPaint) + // Keep the text inside the visible canvas. + val textX = backgroundRect.left + textPadding + val textBaseline = min( + backgroundRect.bottom - textPadding, + backgroundRect.top + textPadding + textHeight + ) + + canvas.drawText( + drawableText, + textX, + textBaseline, + textPaint + ) } } + /** + * Updates detections and scales bounding boxes to match the preview. + */ fun setResults( - detectionResults: MutableList, - imageHeight: Int, - imageWidth: Int, + detectionResults: MutableList, + imageHeight: Int, + imageWidth: Int ) { - results = detectionResults - - // PreviewView is in FILL_START mode. So we need to scale up the bounding box to match with - // the size that the captured images will be displayed. - scaleFactor = max(width * 1f / imageWidth, height * 1f / imageHeight) - } + results = detectionResults.toList() + + if ( + imageWidth <= 0 || + imageHeight <= 0 || + width <= 0 || + height <= 0 + ) { + scaleFactor = 1f + } else { + // Match the PreviewView's FILL_START scaling behavior. + scaleFactor = max( + width.toFloat() / imageWidth, + height.toFloat() / imageHeight + ) + } - companion object { - private const val BOUNDING_RECT_TEXT_PADDING = 8 + invalidate() } } From 9ed2a212ebf58f019cd47f77d867e4d99a70a717 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 21:07:34 +0530 Subject: [PATCH 14/17] Enhance TTS and add Raspberry Pi audio support Updated CameraFragment to improve text-to-speech functionality and added Raspberry Pi audio transmission. --- .../fragments/CameraFragment.kt | 386 +++++++++++++----- 1 file changed, 282 insertions(+), 104 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index 4c92acca997..dbed7c2fbc4 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -2,17 +2,8 @@ /* * Copyright 2022 The TensorFlow Authors. All Rights Reserved. * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Licensed under the Apache License, Version 2.0. + * See https://www.apache.org/licenses/LICENSE-2.0 */ package org.tensorflow.lite.examples.objectdetection.fragments @@ -23,6 +14,7 @@ import android.graphics.Bitmap import android.os.Bundle import android.os.SystemClock import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener import android.util.Log import android.view.LayoutInflater import android.view.View @@ -40,30 +32,39 @@ import androidx.camera.lifecycle.ProcessCameraProvider import androidx.core.content.ContextCompat import androidx.fragment.app.Fragment import androidx.navigation.Navigation -import java.util.LinkedList -import java.util.Locale -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors import org.tensorflow.lite.examples.objectdetection.ObjectDetectorHelper import org.tensorflow.lite.examples.objectdetection.R import org.tensorflow.lite.examples.objectdetection.databinding.FragmentCameraBinding import org.tensorflow.lite.task.vision.detector.Detection +import java.io.DataOutputStream +import java.io.File +import java.io.FileInputStream +import java.net.InetSocketAddress +import java.net.Socket +import java.util.LinkedList +import java.util.Locale +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors -class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { +class CameraFragment : + Fragment(), + ObjectDetectorHelper.DetectorListener { companion object { private const val TAG = "ObjectDetection" - // A label must appear in this many consecutive analyzed frames. - private const val REQUIRED_STABLE_FRAMES = 3 + // Raspberry Pi connection settings + private const val PI_HOST = "10.242.215.42" + private const val PI_PORT = 5001 + private const val CONNECT_TIMEOUT_MS = 3000 + private const val SOCKET_TIMEOUT_MS = 5000 + private const val MAX_AUDIO_SIZE = 10 * 1024 * 1024 - // Minimum time between any two announcements. + // Detection speech settings + private const val REQUIRED_STABLE_FRAMES = 3 private const val MIN_ANNOUNCEMENT_GAP_MS = 2000L - - // Minimum delay before announcing the same label again. private const val SAME_LABEL_REPEAT_DELAY_MS = 6000L - - // Minimum confidence required before a detection can be spoken. private const val SPEECH_CONFIDENCE_THRESHOLD = 0.60f } @@ -75,6 +76,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { private lateinit var objectDetectorHelper: ObjectDetectorHelper private lateinit var bitmapBuffer: Bitmap private lateinit var cameraExecutor: ExecutorService + private lateinit var audioExecutor: ExecutorService private var preview: Preview? = null private var imageAnalyzer: ImageAnalysis? = null @@ -84,6 +86,10 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { // Text-to-Speech private var textToSpeech: TextToSpeech? = null private var ttsReady = false + private var speechPending = false + + // Synthesized audio files, indexed by utterance ID + private val audioFiles = ConcurrentHashMap() // Stable detection tracking private var candidateLabel = "" @@ -92,6 +98,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { // Announcement tracking private var lastSpokenLabel = "" private var lastAnnouncementTime = 0L + private var utteranceCounter = 0 override fun onResume() { super.onResume() @@ -107,13 +114,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } override fun onDestroyView() { - // Stop speech and release the TTS engine. - textToSpeech?.stop() - textToSpeech?.shutdown() - textToSpeech = null - ttsReady = false - - // Stop camera analysis. + // Stop camera analysis before releasing the view. imageAnalyzer?.clearAnalyzer() cameraProvider?.unbindAll() @@ -122,11 +123,27 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { camera = null cameraProvider = null - // Release the background executor. + // Stop TTS and release the engine. + textToSpeech?.stop() + textToSpeech?.shutdown() + textToSpeech = null + ttsReady = false + speechPending = false + + // Release generated files. + audioFiles.values.forEach { file -> + runCatching { file.delete() } + } + audioFiles.clear() + if (::cameraExecutor.isInitialized) { cameraExecutor.shutdown() } + if (::audioExecutor.isInitialized) { + audioExecutor.shutdownNow() + } + _fragmentCameraBinding = null super.onDestroyView() } @@ -143,10 +160,14 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } @SuppressLint("MissingPermission") - override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + override fun onViewCreated( + view: View, + savedInstanceState: Bundle? + ) { super.onViewCreated(view, savedInstanceState) cameraExecutor = Executors.newSingleThreadExecutor() + audioExecutor = Executors.newSingleThreadExecutor() initializeTextToSpeech() @@ -169,40 +190,220 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { ttsReady = false textToSpeech = TextToSpeech(requireContext()) { status -> - if (status == TextToSpeech.SUCCESS) { - val result = textToSpeech?.setLanguage(Locale.US) + if (status != TextToSpeech.SUCCESS) { + Log.e(TAG, "TTS initialization failed") + showToast("Text-to-Speech initialization failed.") + return@TextToSpeech + } - ttsReady = result != null && - result != TextToSpeech.LANG_MISSING_DATA && - result != TextToSpeech.LANG_NOT_SUPPORTED + val engine = textToSpeech ?: return@TextToSpeech + val languageResult = engine.setLanguage(Locale.US) - Log.d(TAG, "Text-to-Speech ready: $ttsReady") - } else { - ttsReady = false - Log.e(TAG, "Text-to-Speech initialization failed") + ttsReady = + languageResult != TextToSpeech.LANG_MISSING_DATA && + languageResult != TextToSpeech.LANG_NOT_SUPPORTED + + if (!ttsReady) { + Log.e(TAG, "TTS language is unavailable") + showToast("English speech is unavailable.") + return@TextToSpeech } + + engine.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + + override fun onStart(utteranceId: String?) { + Log.d(TAG, "Speech synthesis started: $utteranceId") + } + + override fun onDone(utteranceId: String?) { + if (utteranceId == null) return + + val file = audioFiles[utteranceId] ?: return + + if (!::audioExecutor.isInitialized || + audioExecutor.isShutdown + ) { + audioFiles.remove(utteranceId) + file.delete() + return + } + + try { + audioExecutor.execute { + try { + sendAudioToPi(file) + } catch (e: Exception) { + Log.e( + TAG, + "Failed to transmit synthesized audio", + e + ) + showToast( + "Could not send audio to Raspberry Pi." + ) + } finally { + audioFiles.remove(utteranceId) + file.delete() + + activity?.runOnUiThread { + speechPending = false + } + } + } + } catch (e: Exception) { + audioFiles.remove(utteranceId) + file.delete() + Log.e(TAG, "Audio executor unavailable", e) + } + } + + @Deprecated("Deprecated in Android API") + override fun onError(utteranceId: String?) { + handleSynthesisError(utteranceId) + } + + override fun onError( + utteranceId: String?, + errorCode: Int + ) { + Log.e( + TAG, + "Speech synthesis error: $errorCode" + ) + handleSynthesisError(utteranceId) + } + } + ) + + Log.d(TAG, "TTS ready: $ttsReady") + } + } + + private fun handleSynthesisError(utteranceId: String?) { + if (utteranceId != null) { + audioFiles.remove(utteranceId)?.delete() + } + + activity?.runOnUiThread { + speechPending = false } } /** - * Selects the highest-confidence label for the current frame. - * The label must remain stable for several frames before it is spoken. + * Generate a WAV file locally, then send it to the Raspberry Pi. + * This function should be called on the main thread. + */ + private fun synthesizeAndSend(text: String) { + val engine = textToSpeech + + if (!ttsReady || engine == null || speechPending) { + return + } + + if (!::audioExecutor.isInitialized || audioExecutor.isShutdown) { + return + } + + val utteranceId = "blind_assist_${++utteranceCounter}" + val outputFile = File( + requireContext().cacheDir, + "$utteranceId.wav" + ) + + speechPending = true + audioFiles[utteranceId] = outputFile + + val result = engine.synthesizeToFile( + text, + Bundle(), + outputFile, + utteranceId + ) + + if (result != TextToSpeech.SUCCESS) { + audioFiles.remove(utteranceId) + outputFile.delete() + speechPending = false + Log.e(TAG, "Could not start WAV synthesis") + } + } + + /** + * Protocol expected by audio_receiver.py: + * 4-byte big-endian WAV length, followed by WAV bytes. + * + * Network work runs on audioExecutor, never on the UI thread. + */ + private fun sendAudioToPi(file: File) { + if (!file.exists()) { + throw IllegalStateException("Synthesized WAV file is missing") + } + + val audioLength = file.length() + + if (audioLength <= 0 || audioLength > MAX_AUDIO_SIZE) { + throw IllegalStateException( + "Invalid audio file size: $audioLength" + ) + } + + Socket().use { socket -> + socket.connect( + InetSocketAddress(PI_HOST, PI_PORT), + CONNECT_TIMEOUT_MS + ) + socket.soTimeout = SOCKET_TIMEOUT_MS + socket.tcpNoDelay = true + + DataOutputStream(socket.getOutputStream()).use { output -> + // DataOutputStream writes integers in big-endian order. + output.writeInt(audioLength.toInt()) + + FileInputStream(file).use { input -> + val buffer = ByteArray(8192) + var totalSent = 0L + + while (true) { + val count = input.read(buffer) + if (count == -1) break + + output.write(buffer, 0, count) + totalSent += count + } + + output.flush() + + if (totalSent != audioLength) { + throw IllegalStateException( + "Audio transmission was incomplete" + ) + } + } + } + } + + Log.i(TAG, "Audio sent to Raspberry Pi ($audioLength bytes)") + } + + /** + * Select the highest-confidence label, require stable frames, + * then synthesize speech and transmit it to the Pi. */ private fun announceDetection(results: MutableList?) { - if (!ttsReady || textToSpeech == null) return + if (!ttsReady || speechPending) return val bestCategory = results ?.asSequence() ?.mapNotNull { detection -> detection.categories.maxByOrNull { it.score } } - ?.filter { category -> - category.score >= SPEECH_CONFIDENCE_THRESHOLD && - category.label.isNotBlank() + ?.filter { + it.score >= SPEECH_CONFIDENCE_THRESHOLD && + it.label.isNotBlank() } ?.maxByOrNull { it.score } - // No sufficiently confident detection in this frame. if (bestCategory == null) { candidateLabel = "" candidateCount = 0 @@ -211,7 +412,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { val label = bestCategory.label - // Require the same label in consecutive frames. if (label == candidateLabel) { candidateCount++ } else { @@ -219,18 +419,14 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { candidateCount = 1 } - if (candidateCount < REQUIRED_STABLE_FRAMES) { - return - } + if (candidateCount < REQUIRED_STABLE_FRAMES) return val now = SystemClock.elapsedRealtime() - // Avoid speaking too frequently, even when different objects appear. if (now - lastAnnouncementTime < MIN_ANNOUNCEMENT_GAP_MS) { return } - // Avoid repeating the same label too often. if ( label == lastSpokenLabel && now - lastAnnouncementTime < SAME_LABEL_REPEAT_DELAY_MS @@ -238,23 +434,14 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { return } - val result = textToSpeech?.speak( - "$label detected", - TextToSpeech.QUEUE_FLUSH, - null, - "object_warning" - ) - - if (result == TextToSpeech.SUCCESS) { - lastSpokenLabel = label - lastAnnouncementTime = now - - // Require fresh stable frames before the next announcement. - candidateLabel = "" - candidateCount = 0 + // Reserve the speech slot before starting synthesis. + lastSpokenLabel = label + lastAnnouncementTime = now + candidateLabel = "" + candidateCount = 0 - Log.d(TAG, "Announced object: $label") - } + Log.i(TAG, "Announcing detected object: $label") + synthesizeAndSend("$label detected") } private fun initBottomSheetControls() { @@ -264,7 +451,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { objectDetectorHelper.threshold = (objectDetectorHelper.threshold - 0.1f) .coerceAtLeast(0.1f) - updateControlsUi() } } @@ -275,7 +461,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { objectDetectorHelper.threshold = (objectDetectorHelper.threshold + 0.1f) .coerceAtMost(0.9f) - updateControlsUi() } } @@ -331,9 +516,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } } - override fun onNothingSelected(parent: AdapterView<*>?) { - // No action required. - } + override fun onNothingSelected(parent: AdapterView<*>?) = Unit } fragmentCameraBinding.bottomSheetLayout.spinnerModel @@ -355,9 +538,7 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } } - override fun onNothingSelected(parent: AdapterView<*>?) { - // No action required. - } + override fun onNothingSelected(parent: AdapterView<*>?) = Unit } } @@ -377,7 +558,6 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { binding.bottomSheetLayout.threadsValue.text = objectDetectorHelper.numThreads.toString() - // Reset speech stability when detector settings change. candidateLabel = "" candidateCount = 0 @@ -386,10 +566,10 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } private fun setUpCamera() { - val context = context ?: return + val safeContext = context ?: return val cameraProviderFuture = - ProcessCameraProvider.getInstance(context) + ProcessCameraProvider.getInstance(safeContext) cameraProviderFuture.addListener( { @@ -398,22 +578,21 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { try { cameraProvider = cameraProviderFuture.get() bindCameraUseCases() - } catch (exc: Exception) { - Log.e(TAG, "Camera initialization failed", exc) - onError("Unable to initialize the camera.") + } catch (e: Exception) { + Log.e(TAG, "Camera initialization failed", e) + showToast("Unable to initialize the camera.") } }, - ContextCompat.getMainExecutor(context) + ContextCompat.getMainExecutor(safeContext) ) } @SuppressLint("UnsafeOptInUsageError") private fun bindCameraUseCases() { - val provider = cameraProvider - ?: run { - onError("Camera provider is unavailable.") - return - } + val provider = cameraProvider ?: run { + showToast("Camera provider is unavailable.") + return + } val binding = _fragmentCameraBinding ?: return @@ -454,8 +633,8 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { } detectObjects(image) - } catch (exc: Exception) { - Log.e(TAG, "Object detection failed", exc) + } catch (e: Exception) { + Log.e(TAG, "Object detection failed", e) } finally { image.close() } @@ -472,12 +651,10 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { imageAnalyzer ) - preview?.setSurfaceProvider( - binding.viewFinder.surfaceProvider - ) - } catch (exc: Exception) { - Log.e(TAG, "Use case binding failed", exc) - onError("Unable to start the camera preview.") + preview?.setSurfaceProvider(binding.viewFinder.surfaceProvider) + } catch (e: Exception) { + Log.e(TAG, "Use case binding failed", e) + showToast("Unable to start the camera preview.") } } @@ -486,11 +663,9 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) - val imageRotation = image.imageInfo.rotationDegrees - objectDetectorHelper.detect( bitmapBuffer, - imageRotation + image.imageInfo.rotationDegrees ) } @@ -517,23 +692,26 @@ class CameraFragment : Fragment(), ObjectDetectorHelper.DetectorListener { String.format(Locale.US, "%d ms", inferenceTime) binding.overlay.setResults( - results ?: LinkedList(), + results ?: LinkedList(), imageHeight, imageWidth ) binding.overlay.invalidate() - announceDetection(results) } } override fun onError(error: String) { + showToast(error) + } + + private fun showToast(message: String) { activity?.runOnUiThread { context?.let { safeContext -> Toast.makeText( safeContext, - error, + message, Toast.LENGTH_SHORT ).show() } From e3c1b154bb3b977f9e334fcdc89580efd591f3a7 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 21:15:36 +0530 Subject: [PATCH 15/17] Clean up CameraFragment by removing unused code Removed unused variables and methods related to text-to-speech and camera lifecycle management. --- .../fragments/CameraFragment.kt | 624 ------------------ 1 file changed, 624 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index dbed7c2fbc4..a345d8d7550 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -94,627 +94,3 @@ class CameraFragment : // Stable detection tracking private var candidateLabel = "" private var candidateCount = 0 - - // Announcement tracking - private var lastSpokenLabel = "" - private var lastAnnouncementTime = 0L - private var utteranceCounter = 0 - - override fun onResume() { - super.onResume() - - if (!PermissionsFragment.hasPermissions(requireContext())) { - Navigation.findNavController( - requireActivity(), - R.id.fragment_container - ).navigate( - CameraFragmentDirections.actionCameraToPermissions() - ) - } - } - - override fun onDestroyView() { - // Stop camera analysis before releasing the view. - imageAnalyzer?.clearAnalyzer() - cameraProvider?.unbindAll() - - imageAnalyzer = null - preview = null - camera = null - cameraProvider = null - - // Stop TTS and release the engine. - textToSpeech?.stop() - textToSpeech?.shutdown() - textToSpeech = null - ttsReady = false - speechPending = false - - // Release generated files. - audioFiles.values.forEach { file -> - runCatching { file.delete() } - } - audioFiles.clear() - - if (::cameraExecutor.isInitialized) { - cameraExecutor.shutdown() - } - - if (::audioExecutor.isInitialized) { - audioExecutor.shutdownNow() - } - - _fragmentCameraBinding = null - super.onDestroyView() - } - - override fun onCreateView( - inflater: LayoutInflater, - container: ViewGroup?, - savedInstanceState: Bundle? - ): View { - _fragmentCameraBinding = - FragmentCameraBinding.inflate(inflater, container, false) - - return fragmentCameraBinding.root - } - - @SuppressLint("MissingPermission") - override fun onViewCreated( - view: View, - savedInstanceState: Bundle? - ) { - super.onViewCreated(view, savedInstanceState) - - cameraExecutor = Executors.newSingleThreadExecutor() - audioExecutor = Executors.newSingleThreadExecutor() - - initializeTextToSpeech() - - objectDetectorHelper = ObjectDetectorHelper( - context = requireContext(), - objectDetectorListener = this - ) - - fragmentCameraBinding.viewFinder.post { - if (_fragmentCameraBinding != null) { - setUpCamera() - } - } - - initBottomSheetControls() - updateControlsUi() - } - - private fun initializeTextToSpeech() { - ttsReady = false - - textToSpeech = TextToSpeech(requireContext()) { status -> - if (status != TextToSpeech.SUCCESS) { - Log.e(TAG, "TTS initialization failed") - showToast("Text-to-Speech initialization failed.") - return@TextToSpeech - } - - val engine = textToSpeech ?: return@TextToSpeech - val languageResult = engine.setLanguage(Locale.US) - - ttsReady = - languageResult != TextToSpeech.LANG_MISSING_DATA && - languageResult != TextToSpeech.LANG_NOT_SUPPORTED - - if (!ttsReady) { - Log.e(TAG, "TTS language is unavailable") - showToast("English speech is unavailable.") - return@TextToSpeech - } - - engine.setOnUtteranceProgressListener( - object : UtteranceProgressListener() { - - override fun onStart(utteranceId: String?) { - Log.d(TAG, "Speech synthesis started: $utteranceId") - } - - override fun onDone(utteranceId: String?) { - if (utteranceId == null) return - - val file = audioFiles[utteranceId] ?: return - - if (!::audioExecutor.isInitialized || - audioExecutor.isShutdown - ) { - audioFiles.remove(utteranceId) - file.delete() - return - } - - try { - audioExecutor.execute { - try { - sendAudioToPi(file) - } catch (e: Exception) { - Log.e( - TAG, - "Failed to transmit synthesized audio", - e - ) - showToast( - "Could not send audio to Raspberry Pi." - ) - } finally { - audioFiles.remove(utteranceId) - file.delete() - - activity?.runOnUiThread { - speechPending = false - } - } - } - } catch (e: Exception) { - audioFiles.remove(utteranceId) - file.delete() - Log.e(TAG, "Audio executor unavailable", e) - } - } - - @Deprecated("Deprecated in Android API") - override fun onError(utteranceId: String?) { - handleSynthesisError(utteranceId) - } - - override fun onError( - utteranceId: String?, - errorCode: Int - ) { - Log.e( - TAG, - "Speech synthesis error: $errorCode" - ) - handleSynthesisError(utteranceId) - } - } - ) - - Log.d(TAG, "TTS ready: $ttsReady") - } - } - - private fun handleSynthesisError(utteranceId: String?) { - if (utteranceId != null) { - audioFiles.remove(utteranceId)?.delete() - } - - activity?.runOnUiThread { - speechPending = false - } - } - - /** - * Generate a WAV file locally, then send it to the Raspberry Pi. - * This function should be called on the main thread. - */ - private fun synthesizeAndSend(text: String) { - val engine = textToSpeech - - if (!ttsReady || engine == null || speechPending) { - return - } - - if (!::audioExecutor.isInitialized || audioExecutor.isShutdown) { - return - } - - val utteranceId = "blind_assist_${++utteranceCounter}" - val outputFile = File( - requireContext().cacheDir, - "$utteranceId.wav" - ) - - speechPending = true - audioFiles[utteranceId] = outputFile - - val result = engine.synthesizeToFile( - text, - Bundle(), - outputFile, - utteranceId - ) - - if (result != TextToSpeech.SUCCESS) { - audioFiles.remove(utteranceId) - outputFile.delete() - speechPending = false - Log.e(TAG, "Could not start WAV synthesis") - } - } - - /** - * Protocol expected by audio_receiver.py: - * 4-byte big-endian WAV length, followed by WAV bytes. - * - * Network work runs on audioExecutor, never on the UI thread. - */ - private fun sendAudioToPi(file: File) { - if (!file.exists()) { - throw IllegalStateException("Synthesized WAV file is missing") - } - - val audioLength = file.length() - - if (audioLength <= 0 || audioLength > MAX_AUDIO_SIZE) { - throw IllegalStateException( - "Invalid audio file size: $audioLength" - ) - } - - Socket().use { socket -> - socket.connect( - InetSocketAddress(PI_HOST, PI_PORT), - CONNECT_TIMEOUT_MS - ) - socket.soTimeout = SOCKET_TIMEOUT_MS - socket.tcpNoDelay = true - - DataOutputStream(socket.getOutputStream()).use { output -> - // DataOutputStream writes integers in big-endian order. - output.writeInt(audioLength.toInt()) - - FileInputStream(file).use { input -> - val buffer = ByteArray(8192) - var totalSent = 0L - - while (true) { - val count = input.read(buffer) - if (count == -1) break - - output.write(buffer, 0, count) - totalSent += count - } - - output.flush() - - if (totalSent != audioLength) { - throw IllegalStateException( - "Audio transmission was incomplete" - ) - } - } - } - } - - Log.i(TAG, "Audio sent to Raspberry Pi ($audioLength bytes)") - } - - /** - * Select the highest-confidence label, require stable frames, - * then synthesize speech and transmit it to the Pi. - */ - private fun announceDetection(results: MutableList?) { - if (!ttsReady || speechPending) return - - val bestCategory = results - ?.asSequence() - ?.mapNotNull { detection -> - detection.categories.maxByOrNull { it.score } - } - ?.filter { - it.score >= SPEECH_CONFIDENCE_THRESHOLD && - it.label.isNotBlank() - } - ?.maxByOrNull { it.score } - - if (bestCategory == null) { - candidateLabel = "" - candidateCount = 0 - return - } - - val label = bestCategory.label - - if (label == candidateLabel) { - candidateCount++ - } else { - candidateLabel = label - candidateCount = 1 - } - - if (candidateCount < REQUIRED_STABLE_FRAMES) return - - val now = SystemClock.elapsedRealtime() - - if (now - lastAnnouncementTime < MIN_ANNOUNCEMENT_GAP_MS) { - return - } - - if ( - label == lastSpokenLabel && - now - lastAnnouncementTime < SAME_LABEL_REPEAT_DELAY_MS - ) { - return - } - - // Reserve the speech slot before starting synthesis. - lastSpokenLabel = label - lastAnnouncementTime = now - candidateLabel = "" - candidateCount = 0 - - Log.i(TAG, "Announcing detected object: $label") - synthesizeAndSend("$label detected") - } - - private fun initBottomSheetControls() { - fragmentCameraBinding.bottomSheetLayout.thresholdMinus - .setOnClickListener { - if (objectDetectorHelper.threshold > 0.1f) { - objectDetectorHelper.threshold = - (objectDetectorHelper.threshold - 0.1f) - .coerceAtLeast(0.1f) - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.thresholdPlus - .setOnClickListener { - if (objectDetectorHelper.threshold < 0.9f) { - objectDetectorHelper.threshold = - (objectDetectorHelper.threshold + 0.1f) - .coerceAtMost(0.9f) - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.maxResultsMinus - .setOnClickListener { - if (objectDetectorHelper.maxResults > 1) { - objectDetectorHelper.maxResults-- - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.maxResultsPlus - .setOnClickListener { - if (objectDetectorHelper.maxResults < 5) { - objectDetectorHelper.maxResults++ - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.threadsMinus - .setOnClickListener { - if (objectDetectorHelper.numThreads > 1) { - objectDetectorHelper.numThreads-- - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.threadsPlus - .setOnClickListener { - if (objectDetectorHelper.numThreads < 4) { - objectDetectorHelper.numThreads++ - updateControlsUi() - } - } - - fragmentCameraBinding.bottomSheetLayout.spinnerDelegate - .setSelection(0, false) - - fragmentCameraBinding.bottomSheetLayout.spinnerDelegate - .onItemSelectedListener = - object : AdapterView.OnItemSelectedListener { - - override fun onItemSelected( - parent: AdapterView<*>?, - view: View?, - position: Int, - id: Long - ) { - if (objectDetectorHelper.currentDelegate != position) { - objectDetectorHelper.currentDelegate = position - updateControlsUi() - } - } - - override fun onNothingSelected(parent: AdapterView<*>?) = Unit - } - - fragmentCameraBinding.bottomSheetLayout.spinnerModel - .setSelection(0, false) - - fragmentCameraBinding.bottomSheetLayout.spinnerModel - .onItemSelectedListener = - object : AdapterView.OnItemSelectedListener { - - override fun onItemSelected( - parent: AdapterView<*>?, - view: View?, - position: Int, - id: Long - ) { - if (objectDetectorHelper.currentModel != position) { - objectDetectorHelper.currentModel = position - updateControlsUi() - } - } - - override fun onNothingSelected(parent: AdapterView<*>?) = Unit - } - } - - private fun updateControlsUi() { - val binding = _fragmentCameraBinding ?: return - - binding.bottomSheetLayout.maxResultsValue.text = - objectDetectorHelper.maxResults.toString() - - binding.bottomSheetLayout.thresholdValue.text = - String.format( - Locale.US, - "%.2f", - objectDetectorHelper.threshold - ) - - binding.bottomSheetLayout.threadsValue.text = - objectDetectorHelper.numThreads.toString() - - candidateLabel = "" - candidateCount = 0 - - objectDetectorHelper.clearObjectDetector() - binding.overlay.clear() - } - - private fun setUpCamera() { - val safeContext = context ?: return - - val cameraProviderFuture = - ProcessCameraProvider.getInstance(safeContext) - - cameraProviderFuture.addListener( - { - if (_fragmentCameraBinding == null) return@addListener - - try { - cameraProvider = cameraProviderFuture.get() - bindCameraUseCases() - } catch (e: Exception) { - Log.e(TAG, "Camera initialization failed", e) - showToast("Unable to initialize the camera.") - } - }, - ContextCompat.getMainExecutor(safeContext) - ) - } - - @SuppressLint("UnsafeOptInUsageError") - private fun bindCameraUseCases() { - val provider = cameraProvider ?: run { - showToast("Camera provider is unavailable.") - return - } - - val binding = _fragmentCameraBinding ?: return - - val cameraSelector = CameraSelector.Builder() - .requireLensFacing(CameraSelector.LENS_FACING_BACK) - .build() - - preview = Preview.Builder() - .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(binding.viewFinder.display.rotation) - .build() - - imageAnalyzer = ImageAnalysis.Builder() - .setTargetAspectRatio(AspectRatio.RATIO_4_3) - .setTargetRotation(binding.viewFinder.display.rotation) - .setBackpressureStrategy( - ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST - ) - .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) - .build() - .also { analyzer -> - analyzer.setAnalyzer(cameraExecutor) { image -> - try { - if (_fragmentCameraBinding == null) { - return@setAnalyzer - } - - if ( - !::bitmapBuffer.isInitialized || - bitmapBuffer.width != image.width || - bitmapBuffer.height != image.height - ) { - bitmapBuffer = Bitmap.createBitmap( - image.width, - image.height, - Bitmap.Config.ARGB_8888 - ) - } - - detectObjects(image) - } catch (e: Exception) { - Log.e(TAG, "Object detection failed", e) - } finally { - image.close() - } - } - } - - provider.unbindAll() - - try { - camera = provider.bindToLifecycle( - this, - cameraSelector, - preview, - imageAnalyzer - ) - - preview?.setSurfaceProvider(binding.viewFinder.surfaceProvider) - } catch (e: Exception) { - Log.e(TAG, "Use case binding failed", e) - showToast("Unable to start the camera preview.") - } - } - - private fun detectObjects(image: ImageProxy) { - if (!::bitmapBuffer.isInitialized) return - - bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) - - objectDetectorHelper.detect( - bitmapBuffer, - image.imageInfo.rotationDegrees - ) - } - - override fun onConfigurationChanged(newConfig: Configuration) { - super.onConfigurationChanged(newConfig) - - _fragmentCameraBinding?.let { binding -> - imageAnalyzer?.targetRotation = - binding.viewFinder.display.rotation - } - } - - override fun onResults( - results: MutableList?, - inferenceTime: Long, - imageHeight: Int, - imageWidth: Int - ) { - activity?.runOnUiThread { - val binding = _fragmentCameraBinding - ?: return@runOnUiThread - - binding.bottomSheetLayout.inferenceTimeVal.text = - String.format(Locale.US, "%d ms", inferenceTime) - - binding.overlay.setResults( - results ?: LinkedList(), - imageHeight, - imageWidth - ) - - binding.overlay.invalidate() - announceDetection(results) - } - } - - override fun onError(error: String) { - showToast(error) - } - - private fun showToast(message: String) { - activity?.runOnUiThread { - context?.let { safeContext -> - Toast.makeText( - safeContext, - message, - Toast.LENGTH_SHORT - ).show() - } - } - } -} From 9170be8d9f43c8521eb17582abb91b9d3897985c Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 21:21:30 +0530 Subject: [PATCH 16/17] Implement TTS for object detection announcements Added text-to-speech functionality for object detection announcements, including initialization, synthesis, and audio transmission to Raspberry Pi. --- .../fragments/CameraFragment.kt | 627 ++++++++++++++++++ 1 file changed, 627 insertions(+) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index a345d8d7550..c9e86fceef8 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -94,3 +94,630 @@ class CameraFragment : // Stable detection tracking private var candidateLabel = "" private var candidateCount = 0 + + // Announcement tracking + private var lastSpokenLabel = "" + private var lastAnnouncementTime = 0L + private var utteranceCounter = 0 + + override fun onResume() { + super.onResume() + + if (!PermissionsFragment.hasPermissions(requireContext())) { + Navigation.findNavController( + requireActivity(), + R.id.fragment_container + ).navigate( + CameraFragmentDirections.actionCameraToPermissions() + ) + } + } + + override fun onDestroyView() { + // Stop camera analysis before releasing the view. + imageAnalyzer?.clearAnalyzer() + cameraProvider?.unbindAll() + + imageAnalyzer = null + preview = null + camera = null + cameraProvider = null + + // Stop TTS and release the engine. + textToSpeech?.stop() + textToSpeech?.shutdown() + textToSpeech = null + ttsReady = false + speechPending = false + + // Release generated files. + audioFiles.values.forEach { file -> + runCatching { file.delete() } + } + audioFiles.clear() + + if (::cameraExecutor.isInitialized) { + cameraExecutor.shutdown() + } + + if (::audioExecutor.isInitialized) { + audioExecutor.shutdownNow() + } + + _fragmentCameraBinding = null + super.onDestroyView() + } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle? + ): View { + _fragmentCameraBinding = + FragmentCameraBinding.inflate(inflater, container, false) + + return fragmentCameraBinding.root + } + + @SuppressLint("MissingPermission") + override fun onViewCreated( + view: View, + savedInstanceState: Bundle? + ) { + super.onViewCreated(view, savedInstanceState) + + cameraExecutor = Executors.newSingleThreadExecutor() + audioExecutor = Executors.newSingleThreadExecutor() + + initializeTextToSpeech() + + objectDetectorHelper = ObjectDetectorHelper( + context = requireContext(), + objectDetectorListener = this + ) + + fragmentCameraBinding.viewFinder.post { + if (_fragmentCameraBinding != null) { + setUpCamera() + } + } + + initBottomSheetControls() + updateControlsUi() + } + + private fun initializeTextToSpeech() { + ttsReady = false + + textToSpeech = TextToSpeech(requireContext()) { status -> + if (status != TextToSpeech.SUCCESS) { + Log.e(TAG, "TTS initialization failed") + showToast("Text-to-Speech initialization failed.") + return@TextToSpeech + } + + val engine = textToSpeech ?: return@TextToSpeech + val languageResult = engine.setLanguage(Locale.US) + + ttsReady = + languageResult != TextToSpeech.LANG_MISSING_DATA && + languageResult != TextToSpeech.LANG_NOT_SUPPORTED + + if (!ttsReady) { + Log.e(TAG, "TTS language is unavailable") + showToast("English speech is unavailable.") + return@TextToSpeech + } + + engine.setOnUtteranceProgressListener( + object : UtteranceProgressListener() { + + override fun onStart(utteranceId: String?) { + Log.d(TAG, "Speech synthesis started: $utteranceId") + } + + override fun onDone(utteranceId: String?) { + if (utteranceId == null) return + + val file = audioFiles[utteranceId] ?: return + + if (!::audioExecutor.isInitialized || + audioExecutor.isShutdown + ) { + audioFiles.remove(utteranceId) + file.delete() + return + } + + try { + audioExecutor.execute { + try { + sendAudioToPi(file) + } catch (e: Exception) { + Log.e( + TAG, + "Failed to transmit synthesized audio", + e + ) + showToast( + "Could not send audio to Raspberry Pi." + ) + } finally { + audioFiles.remove(utteranceId) + file.delete() + + activity?.runOnUiThread { + speechPending = false + } + } + } + } catch (e: Exception) { + audioFiles.remove(utteranceId) + file.delete() + Log.e(TAG, "Audio executor unavailable", e) + activity?.runOnUiThread { + speechPending = false + } + } + } + + @Deprecated("Deprecated in Android API") + override fun onError(utteranceId: String?) { + handleSynthesisError(utteranceId) + } + + override fun onError( + utteranceId: String?, + errorCode: Int + ) { + Log.e( + TAG, + "Speech synthesis error: $errorCode" + ) + handleSynthesisError(utteranceId) + } + } + ) + + Log.d(TAG, "TTS ready: $ttsReady") + } + } + + private fun handleSynthesisError(utteranceId: String?) { + if (utteranceId != null) { + audioFiles.remove(utteranceId)?.delete() + } + + activity?.runOnUiThread { + speechPending = false + } + } + + /** + * Generate a WAV file locally, then send it to the Raspberry Pi. + * This function should be called on the main thread. + */ + private fun synthesizeAndSend(text: String) { + val engine = textToSpeech + + if (!ttsReady || engine == null || speechPending) { + return + } + + if (!::audioExecutor.isInitialized || audioExecutor.isShutdown) { + return + } + + val utteranceId = "blind_assist_${++utteranceCounter}" + val outputFile = File( + requireContext().cacheDir, + "$utteranceId.wav" + ) + + speechPending = true + audioFiles[utteranceId] = outputFile + + val result = engine.synthesizeToFile( + text, + Bundle(), + outputFile, + utteranceId + ) + + if (result != TextToSpeech.SUCCESS) { + audioFiles.remove(utteranceId) + outputFile.delete() + speechPending = false + Log.e(TAG, "Could not start WAV synthesis") + } + } + + /** + * Protocol expected by audio_receiver.py: + * 4-byte big-endian WAV length, followed by WAV bytes. + * + * Network work runs on audioExecutor, never on the UI thread. + */ + private fun sendAudioToPi(file: File) { + if (!file.exists()) { + throw IllegalStateException("Synthesized WAV file is missing") + } + + val audioLength = file.length() + + if (audioLength <= 0 || audioLength > MAX_AUDIO_SIZE) { + throw IllegalStateException( + "Invalid audio file size: $audioLength" + ) + } + + Socket().use { socket -> + socket.connect( + InetSocketAddress(PI_HOST, PI_PORT), + CONNECT_TIMEOUT_MS + ) + socket.soTimeout = SOCKET_TIMEOUT_MS + socket.tcpNoDelay = true + + DataOutputStream(socket.getOutputStream()).use { output -> + // DataOutputStream writes integers in big-endian order. + output.writeInt(audioLength.toInt()) + + FileInputStream(file).use { input -> + val buffer = ByteArray(8192) + var totalSent = 0L + + while (true) { + val count = input.read(buffer) + if (count == -1) break + + output.write(buffer, 0, count) + totalSent += count + } + + output.flush() + + if (totalSent != audioLength) { + throw IllegalStateException( + "Audio transmission was incomplete" + ) + } + } + } + } + + Log.i(TAG, "Audio sent to Raspberry Pi ($audioLength bytes)") + } + + /** + * Select the highest-confidence label, require stable frames, + * then synthesize speech and transmit it to the Pi. + */ + private fun announceDetection(results: MutableList?) { + if (!ttsReady || speechPending) return + + val bestCategory = results + ?.asSequence() + ?.mapNotNull { detection -> + detection.categories.maxByOrNull { it.score } + } + ?.filter { + it.score >= SPEECH_CONFIDENCE_THRESHOLD && + it.label.isNotBlank() + } + ?.maxByOrNull { it.score } + + if (bestCategory == null) { + candidateLabel = "" + candidateCount = 0 + return + } + + val label = bestCategory.label + + if (label == candidateLabel) { + candidateCount++ + } else { + candidateLabel = label + candidateCount = 1 + } + + if (candidateCount < REQUIRED_STABLE_FRAMES) return + + val now = SystemClock.elapsedRealtime() + + if (now - lastAnnouncementTime < MIN_ANNOUNCEMENT_GAP_MS) { + return + } + + if ( + label == lastSpokenLabel && + now - lastAnnouncementTime < SAME_LABEL_REPEAT_DELAY_MS + ) { + return + } + + // Reserve the speech slot before starting synthesis. + lastSpokenLabel = label + lastAnnouncementTime = now + candidateLabel = "" + candidateCount = 0 + + Log.i(TAG, "Announcing detected object: $label") + synthesizeAndSend("$label detected") + } + + private fun initBottomSheetControls() { + fragmentCameraBinding.bottomSheetLayout.thresholdMinus + .setOnClickListener { + if (objectDetectorHelper.threshold > 0.1f) { + objectDetectorHelper.threshold = + (objectDetectorHelper.threshold - 0.1f) + .coerceAtLeast(0.1f) + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.thresholdPlus + .setOnClickListener { + if (objectDetectorHelper.threshold < 0.9f) { + objectDetectorHelper.threshold = + (objectDetectorHelper.threshold + 0.1f) + .coerceAtMost(0.9f) + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.maxResultsMinus + .setOnClickListener { + if (objectDetectorHelper.maxResults > 1) { + objectDetectorHelper.maxResults-- + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.maxResultsPlus + .setOnClickListener { + if (objectDetectorHelper.maxResults < 5) { + objectDetectorHelper.maxResults++ + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.threadsMinus + .setOnClickListener { + if (objectDetectorHelper.numThreads > 1) { + objectDetectorHelper.numThreads-- + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.threadsPlus + .setOnClickListener { + if (objectDetectorHelper.numThreads < 4) { + objectDetectorHelper.numThreads++ + updateControlsUi() + } + } + + fragmentCameraBinding.bottomSheetLayout.spinnerDelegate + .setSelection(0, false) + + fragmentCameraBinding.bottomSheetLayout.spinnerDelegate + .onItemSelectedListener = + object : AdapterView.OnItemSelectedListener { + + override fun onItemSelected( + parent: AdapterView<*>?, + view: View?, + position: Int, + id: Long + ) { + if (objectDetectorHelper.currentDelegate != position) { + objectDetectorHelper.currentDelegate = position + updateControlsUi() + } + } + + override fun onNothingSelected(parent: AdapterView<*>?) = Unit + } + + fragmentCameraBinding.bottomSheetLayout.spinnerModel + .setSelection(0, false) + + fragmentCameraBinding.bottomSheetLayout.spinnerModel + .onItemSelectedListener = + object : AdapterView.OnItemSelectedListener { + + override fun onItemSelected( + parent: AdapterView<*>?, + view: View?, + position: Int, + id: Long + ) { + if (objectDetectorHelper.currentModel != position) { + objectDetectorHelper.currentModel = position + updateControlsUi() + } + } + + override fun onNothingSelected(parent: AdapterView<*>?) = Unit + } + } + + private fun updateControlsUi() { + val binding = _fragmentCameraBinding ?: return + + binding.bottomSheetLayout.maxResultsValue.text = + objectDetectorHelper.maxResults.toString() + + binding.bottomSheetLayout.thresholdValue.text = + String.format( + Locale.US, + "%.2f", + objectDetectorHelper.threshold + ) + + binding.bottomSheetLayout.threadsValue.text = + objectDetectorHelper.numThreads.toString() + + candidateLabel = "" + candidateCount = 0 + + objectDetectorHelper.clearObjectDetector() + binding.overlay.clear() + } + + private fun setUpCamera() { + val safeContext = context ?: return + + val cameraProviderFuture = + ProcessCameraProvider.getInstance(safeContext) + + cameraProviderFuture.addListener( + { + if (_fragmentCameraBinding == null) return@addListener + + try { + cameraProvider = cameraProviderFuture.get() + bindCameraUseCases() + } catch (e: Exception) { + Log.e(TAG, "Camera initialization failed", e) + showToast("Unable to initialize the camera.") + } + }, + ContextCompat.getMainExecutor(safeContext) + ) + } + + @SuppressLint("UnsafeOptInUsageError") + private fun bindCameraUseCases() { + val provider = cameraProvider ?: run { + showToast("Camera provider is unavailable.") + return + } + + val binding = _fragmentCameraBinding ?: return + + val cameraSelector = CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build() + + preview = Preview.Builder() + .setTargetAspectRatio(AspectRatio.RATIO_4_3) + .setTargetRotation(binding.viewFinder.display.rotation) + .build() + + imageAnalyzer = ImageAnalysis.Builder() + .setTargetAspectRatio(AspectRatio.RATIO_4_3) + .setTargetRotation(binding.viewFinder.display.rotation) + .setBackpressureStrategy( + ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST + ) + .setOutputImageFormat(OUTPUT_IMAGE_FORMAT_RGBA_8888) + .build() + .also { analyzer -> + analyzer.setAnalyzer(cameraExecutor) { image -> + try { + if (_fragmentCameraBinding == null) { + return@setAnalyzer + } + + if ( + !::bitmapBuffer.isInitialized || + bitmapBuffer.width != image.width || + bitmapBuffer.height != image.height + ) { + bitmapBuffer = Bitmap.createBitmap( + image.width, + image.height, + Bitmap.Config.ARGB_8888 + ) + } + + detectObjects(image) + } catch (e: Exception) { + Log.e(TAG, "Object detection failed", e) + } finally { + image.close() + } + } + } + + provider.unbindAll() + + try { + camera = provider.bindToLifecycle( + this, + cameraSelector, + preview, + imageAnalyzer + ) + + preview?.setSurfaceProvider(binding.viewFinder.surfaceProvider) + } catch (e: Exception) { + Log.e(TAG, "Use case binding failed", e) + showToast("Unable to start the camera preview.") + } + } + + private fun detectObjects(image: ImageProxy) { + if (!::bitmapBuffer.isInitialized) return + + bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer) + + objectDetectorHelper.detect( + bitmapBuffer, + image.imageInfo.rotationDegrees + ) + } + + override fun onConfigurationChanged(newConfig: Configuration) { + super.onConfigurationChanged(newConfig) + + _fragmentCameraBinding?.let { binding -> + imageAnalyzer?.targetRotation = + binding.viewFinder.display.rotation + } + } + + override fun onResults( + results: MutableList?, + inferenceTime: Long, + imageHeight: Int, + imageWidth: Int + ) { + activity?.runOnUiThread { + val binding = _fragmentCameraBinding + ?: return@runOnUiThread + + binding.bottomSheetLayout.inferenceTimeVal.text = + String.format(Locale.US, "%d ms", inferenceTime) + + binding.overlay.setResults( + results ?: LinkedList(), + imageHeight, + imageWidth + ) + + binding.overlay.invalidate() + announceDetection(results) + } + } + + override fun onError(error: String) { + showToast(error) + } + + private fun showToast(message: String) { + activity?.runOnUiThread { + context?.let { safeContext -> + Toast.makeText( + safeContext, + message, + Toast.LENGTH_SHORT + ).show() + } + } + } +} From 34be87dbd58b1566e63fb89b469107caba7cb492 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Sat, 17 Oct 2026 21:59:18 +0530 Subject: [PATCH 17/17] Hide overlays and update detection announcement format Hide visual overlays for bounding boxes and labels while keeping detection active. Update announcement message format for detected objects. --- .../objectdetection/fragments/CameraFragment.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt index c9e86fceef8..2ea0b972860 100644 --- a/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt +++ b/lite/examples/object_detection/android/app/src/main/java/org/tensorflow/lite/examples/objectdetection/fragments/CameraFragment.kt @@ -169,6 +169,10 @@ class CameraFragment : cameraExecutor = Executors.newSingleThreadExecutor() audioExecutor = Executors.newSingleThreadExecutor() + // Hide bounding boxes and object labels for the demonstration. + // Detection still runs; only the visual overlay is hidden. + fragmentCameraBinding.overlay.visibility = View.GONE + initializeTextToSpeech() objectDetectorHelper = ObjectDetectorHelper( @@ -443,8 +447,11 @@ class CameraFragment : candidateLabel = "" candidateCount = 0 - Log.i(TAG, "Announcing detected object: $label") - synthesizeAndSend("$label detected") + val spokenLabel = label.replaceFirstChar { it.titlecase(Locale.US) } + val message = "$spokenLabel is detected." + + Log.i(TAG, "Announcing detected object: $message") + synthesizeAndSend(message) } private fun initBottomSheetControls() {