1 files changed, 281 insertions, 0 deletions

diff --git a/tflite/src/main/java/com/example/android/camerax/tflite/CameraActivity.kt b/tflite/src/main/java/com/example/android/camerax/tflite/CameraActivity.kt
new file mode 100644
index 0000000..cde9778
--- /dev/null
+++ b/tflite/src/main/java/com/example/android/camerax/tflite/CameraActivity.kt
@@ -0,0 +1,281 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.example.android.camerax.tflite
+
+import android.Manifest
+import android.annotation.SuppressLint
+import android.content.Context
+import android.content.pm.PackageManager
+import android.graphics.Bitmap
+import android.graphics.Color
+import android.graphics.Matrix
+import android.graphics.RectF
+import android.os.Bundle
+import android.util.Log
+import android.util.Size
+import android.view.View
+import android.view.ViewGroup
+import androidx.appcompat.app.AppCompatActivity
+import androidx.camera.core.AspectRatio
+import androidx.camera.core.CameraSelector
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.Preview
+import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.core.app.ActivityCompat
+import androidx.core.content.ContextCompat
+import androidx.lifecycle.LifecycleOwner
+import com.android.example.camerax.tflite.databinding.ActivityCameraBinding
+import java.net.DatagramPacket
+import java.net.DatagramSocket
+import java.net.InetSocketAddress
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import kotlin.random.Random
+
+
+/** Activity that displays the camera and performs object detection on the incoming frames */
+class CameraActivity : AppCompatActivity() {
+
+    private lateinit var activityCameraBinding: ActivityCameraBinding
+
+    private lateinit var bitmapBuffer: Bitmap
+
+    private val executor = Executors.newSingleThreadExecutor()
+    private val permissions = listOf(Manifest.permission.CAMERA)
+    private val permissionsRequestCode = Random.nextInt(0, 10000)
+
+    private var lensFacing: Int = CameraSelector.LENS_FACING_BACK
+    private val isFrontFacing get() = lensFacing == CameraSelector.LENS_FACING_FRONT
+
+    private var pauseAnalysis = false
+    private var imageRotationDegrees: Int = 0
+    private var socket: DatagramSocket = DatagramSocket()
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        activityCameraBinding = ActivityCameraBinding.inflate(layoutInflater)
+        setContentView(activityCameraBinding.root)
+    }
+
+    override fun onDestroy() {
+
+        // Terminate all outstanding analyzing jobs (if there is any).
+        executor.apply {
+            shutdown()
+            awaitTermination(1000, TimeUnit.MILLISECONDS)
+        }
+        super.onDestroy()
+    }
+
+    /** Declare and bind preview and analysis use cases */
+    @SuppressLint("UnsafeExperimentalUsageError")
+    private fun bindCameraUseCases() = activityCameraBinding.viewFinder.post {
+
+        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)
+        cameraProviderFuture.addListener ({
+
+            // Camera provider is now guaranteed to be available
+            val cameraProvider = cameraProviderFuture.get()
+
+            // Set up the view finder use case to display camera preview
+            val preview = Preview.Builder()
+                .setTargetAspectRatio(AspectRatio.RATIO_4_3)
+                .setTargetRotation(activityCameraBinding.viewFinder.display.rotation)
+                .build()
+
+            // Set up the image analysis use case which will process frames in real time
+            val imageAnalysis = ImageAnalysis.Builder()
+                .setTargetResolution(Size(448, 236))
+                .setTargetRotation(activityCameraBinding.viewFinder.display.rotation)
+                .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+                .setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
+                .build()
+
+            var frameCounter = 0
+            var lastFpsTimestamp = System.currentTimeMillis()
+
+            imageAnalysis.setAnalyzer(executor, ImageAnalysis.Analyzer { image ->
+                if (!::bitmapBuffer.isInitialized) {
+                    // The image rotation and RGB image buffer are initialized only once
+                    // the analyzer has started running
+                    imageRotationDegrees = image.imageInfo.rotationDegrees
+                    bitmapBuffer = Bitmap.createBitmap(
+                        image.width, image.height, Bitmap.Config.ARGB_8888)
+                }
+
+                // Early exit: image analysis is in paused state
+                if (pauseAnalysis) {
+                    image.close()
+                    return@Analyzer
+                }
+
+                // Copy out RGB bits to our shared buffer
+                image.use { bitmapBuffer.copyPixelsFromBuffer(image.planes[0].buffer)  }
+
+//              val CAM_WIDTH = 858
+//              val CAM_HEIGHT = 480
+                val CAM_WIDTH = 480
+                val CAM_HEIGHT = 360
+
+                val DISPLAY_WIDTH = 448
+                val DISPLAY_HEIGHT = 160
+                val DISPLAY_VHEIGHT = 236
+
+                val scaledBitmap = Bitmap.createScaledBitmap(bitmapBuffer, DISPLAY_WIDTH, DISPLAY_VHEIGHT, true)
+
+                val scratch = IntArray((2 + DISPLAY_VHEIGHT) * DISPLAY_WIDTH)
+                val output = UByteArray(10 + DISPLAY_HEIGHT * DISPLAY_WIDTH / 8)
+                output[1] = 0x12u
+                output[4] = 0x23u
+                var offset = 10
+/*
+                for ( row in 0..DISPLAY_VHEIGHT - 1) {
+                    val zeile = (row * bitmapBuffer.height ) / DISPLAY_VHEIGHT
+                    for (column in 0..DISPLAY_WIDTH - 1) {
+                        val spalte = (column * bitmapBuffer.width ) / DISPLAY_WIDTH
+                        val pixel = bitmapBuffer.getPixel(spalte, zeile)
+                        scratch[row * DISPLAY_WIDTH + column] = Color.red(pixel) * 19535 + Color.green(pixel) * 38470 + Color.blue(pixel) * 7448
+                    }
+                }
+*/
+                scaledBitmap.getPixels(scratch, 0, DISPLAY_WIDTH, 0, 0, DISPLAY_WIDTH, DISPLAY_VHEIGHT)
+                for (off in 0..DISPLAY_VHEIGHT * DISPLAY_WIDTH)
+                    scratch[off] = Color.red(scratch[off]) * 19535 + Color.green(scratch[off]) * 38470 + Color.blue(scratch[off]) * 7448
+
+                var acc = 0
+                var accv = 0
+
+                for ( row in 0.. DISPLAY_VHEIGHT - 1)
+                    for ( column in 0..DISPLAY_WIDTH - 1) {
+                        val pixel = scratch[row * DISPLAY_WIDTH + column]
+                        val bwpixel = if (pixel < 0x810000) 0 else 0xFFFFFF
+
+                        if (row % 12 < 8) {
+                            acc = (acc shl 1) + (bwpixel shr 23)
+                            if (++accv == 8) {
+                                output[offset++] = acc.toUByte()
+                                acc = 0
+                                accv = 0
+                            }
+                        }
+
+                        val err = (pixel - bwpixel) / 42
+                        fun AddSatShift(xoff : Int, yoff: Int, shift : Int) {
+                            val pixelold = scratch[(row + yoff) * DISPLAY_WIDTH + column + xoff ]
+                            var r = pixelold + (err shl (16 - shift))
+                            if ( r < 0 ) r = 0
+                            if ( r > 0xFFFFFF) r = 0xFFFFFF
+                            scratch[(row + yoff) * DISPLAY_WIDTH + column + xoff ] = r
+                        }
+
+                        AddSatShift(0, 1, 13)
+                        AddSatShift(0, 2, 14)
+                        if (column > 0) {
+                            AddSatShift(-1, 1, 14)
+                            AddSatShift(-1, 2, 15)
+                        }
+
+                        if (column > 1) {
+                            AddSatShift(-2, 1, 15)
+                            AddSatShift(-2, 2, 16)
+                        }
+
+                        if (column < DISPLAY_WIDTH - 1) {
+                            AddSatShift( 1, 0, 13)
+                            AddSatShift( 1, 1, 14)
+                            AddSatShift( 1, 2, 15)
+                        }
+
+                        if (column < DISPLAY_WIDTH - 2) {
+                            AddSatShift( 2, 0, 14)
+                            AddSatShift( 2, 1, 15)
+                            AddSatShift( 2, 2, 16)
+                        }
+                    }
+                val address = InetSocketAddress("172.23.42.29", 2342 )
+//                val address = InetSocketAddress("192.168.178.69", 2342 )
+                try {
+                    socket.send(DatagramPacket(output.toByteArray(), offset, address))
+                } catch (e: Exception) {
+                    // Ignore network exceptions
+                }
+
+                // Compute the FPS of the entire pipeline
+                val frameCount = 10
+                if (++frameCounter % frameCount == 0) {
+                    frameCounter = 0
+                    val now = System.currentTimeMillis()
+                    val delta = now - lastFpsTimestamp
+                    val fps = 1000 * frameCount.toFloat() / delta
+//                    Log.d(TAG, "FPS: ${"%.02f".format(fps)} " + bitmapBuffer.width + " x " + bitmapBuffer.height)
+
+                    activityCameraBinding.viewFinder.post {
+                        activityCameraBinding.textPrediction.text = "FPS: ${"%.02f".format(fps)}"
+                    }
+
+                    lastFpsTimestamp = now
+                }
+            })
+
+            // Create a new camera selector each time, enforcing lens facing
+            val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build()
+
+            // Apply declared configs to CameraX using the same lifecycle owner
+            cameraProvider.unbindAll()
+            cameraProvider.bindToLifecycle(
+                this as LifecycleOwner, cameraSelector, preview, imageAnalysis)
+
+            // Use the camera object to link our preview use case with the view
+            preview.setSurfaceProvider(activityCameraBinding.viewFinder.surfaceProvider)
+
+        }, ContextCompat.getMainExecutor(this))
+    }
+
+    override fun onResume() {
+        super.onResume()
+
+        // Request permissions each time the app resumes, since they can be revoked at any time
+        if (!hasPermissions(this)) {
+            ActivityCompat.requestPermissions(
+                this, permissions.toTypedArray(), permissionsRequestCode)
+        } else {
+            bindCameraUseCases()
+        }
+    }
+
+    override fun onRequestPermissionsResult(
+        requestCode: Int,
+        permissions: Array<out String>,
+        grantResults: IntArray
+    ) {
+        super.onRequestPermissionsResult(requestCode, permissions, grantResults)
+        if (requestCode == permissionsRequestCode && hasPermissions(this)) {
+            bindCameraUseCases()
+        } else {
+            finish() // If we don't have the required permissions, we can't run
+        }
+    }
+
+    /** Convenience method used to check if all permissions required by this app are granted */
+    private fun hasPermissions(context: Context) = permissions.all {
+        ContextCompat.checkSelfPermission(context, it) == PackageManager.PERMISSION_GRANTED
+    }
+
+    companion object {
+        private val TAG = CameraActivity::class.java.simpleName
+    }
+}