Skip to content

Refactor media-projection & screenshot handling, improve accessibility and command parsing, add app mappings and tests #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Android-PowerUser merged 1 commit into from
Mar 26, 2026
Merged

Refactor media-projection & screenshot handling, improve accessibility and command parsing, add app mappings and tests #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
602 changes: 315 additions & 287 deletions app/src/main/kotlin/com/google/ai/sample/MainActivity.kt
View file
Open in desktop

Large diffs are not rendered by default.

17 changes: 8 additions & 9 deletions app/src/main/kotlin/com/google/ai/sample/ScreenCaptureService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.google.ai.sample

import android.app.Activity // Make sure this import is present
import android.app.Activity
import android.app.Notification
import android.app.NotificationChannel
import android.app.NotificationManager
Expand All @@ -15,7 +15,7 @@ import android.hardware.display.VirtualDisplay
import android.media.ImageReader
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.net.Uri // Added for broadcasting URI
import android.net.Uri
import android.os.Build
import android.os.Environment
import android.os.Handler
Expand All @@ -27,12 +27,11 @@ import android.view.WindowManager
import android.widget.Toast
import com.google.ai.client.generativeai.GenerativeModel
import com.google.ai.client.generativeai.type.Content
import com.google.ai.client.generativeai.type.ImagePart // For instance check
import com.google.ai.client.generativeai.type.FunctionCallPart // For logging AI response
import com.google.ai.client.generativeai.type.FunctionResponsePart // For logging AI response
import com.google.ai.client.generativeai.type.BlobPart // For logging AI response
import com.google.ai.client.generativeai.type.TextPart // For logging AI response
// Removed duplicate TextPart import
import com.google.ai.client.generativeai.type.ImagePart
import com.google.ai.client.generativeai.type.FunctionCallPart
import com.google.ai.client.generativeai.type.FunctionResponsePart
import com.google.ai.client.generativeai.type.BlobPart
import com.google.ai.client.generativeai.type.TextPart
import com.google.ai.sample.feature.multimodal.dtos.ContentDto
import com.google.ai.sample.feature.multimodal.dtos.toSdk
import com.google.ai.sample.service.AiCallRequestExtras
Expand Down Expand Up @@ -73,7 +72,7 @@ class ScreenCaptureService : Service() {
const val ACTION_START_CAPTURE = "com.google.ai.sample.START_CAPTURE"
const val ACTION_TAKE_SCREENSHOT = "com.google.ai.sample.TAKE_SCREENSHOT" // New action
const val ACTION_STOP_CAPTURE = "com.google.ai.sample.STOP_CAPTURE" // New action
const val ACTION_KEEP_ALIVE_FOR_WEBRTC = "com.google.ai.sample.KEEP_ALIVE_FOR_WEBRTC" // Added for Task 4
const val ACTION_KEEP_ALIVE_FOR_WEBRTC = "com.google.ai.sample.KEEP_ALIVE_FOR_WEBRTC"
const val EXTRA_RESULT_CODE = "result_code"
const val EXTRA_RESULT_DATA = "result_data"
const val EXTRA_TAKE_SCREENSHOT_ON_START = "take_screenshot_on_start"
Expand Down
122 changes: 50 additions & 72 deletions app/src/main/kotlin/com/google/ai/sample/ScreenOperatorAccessibilityService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.google.ai.sample

import com.google.ai.sample.MainActivity // Added import
import com.google.ai.sample.MainActivity
import android.accessibilityservice.AccessibilityService
import android.accessibilityservice.AccessibilityServiceInfo
import android.accessibilityservice.GestureDescription
Expand Down Expand Up @@ -138,12 +138,17 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
}

// serviceInstance is the static reference to the service
serviceInstance!!.commandQueue.add(command)
Log.d(TAG, "Command $command added to queue. Queue size: ${serviceInstance!!.commandQueue.size}")
val instance = serviceInstance
if (instance == null) {
Log.e(TAG, "Service instance became null before queueing command")
return
}
instance.commandQueue.add(command)
Log.d(TAG, "Command $command added to queue. Queue size: ${instance.commandQueue.size}")

// Ensure processCommandQueue is called on the service's handler thread
serviceInstance!!.handler.post {
serviceInstance!!.processCommandQueue()
instance.handler.post {
instance.processCommandQueue()
}
}

Expand Down Expand Up @@ -173,6 +178,24 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {

// App name to package mapper
private lateinit var appNamePackageMapper: AppNamePackageMapper

private fun currentRootNodeOrHandleMissing(
operation: String,
scheduleNext: Boolean,
showUserMessage: Boolean = true
): AccessibilityNodeInfo? {
val currentRootNode = rootNode
if (currentRootNode == null) {
Log.e(TAG, "Root node is null, cannot $operation")
if (showUserMessage) {
showToast("Error: Root node is not available", true)
}
if (scheduleNext) {
scheduleNextCommandProcessing()
}
}
return currentRootNode
}

override fun onServiceConnected() {
super.onServiceConnected()
Expand Down Expand Up @@ -607,15 +630,9 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
// Refresh the root node
refreshRootNode()

// Check if root node is available
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot write text")
showToast("Error: Root node is not available", true)
return
}

val currentRootNode = currentRootNodeOrHandleMissing("write text", scheduleNext = false) ?: return
// Find the focused node (which should be an editable text field)
val focusedNode = findFocusedEditableNode(rootNode!!)
val focusedNode = findFocusedEditableNode(currentRootNode)

if (focusedNode != null) {
Log.d(TAG, "Found focused editable node")
Expand Down Expand Up @@ -644,7 +661,7 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
showToast("No focused text field found, trying to find editable fields", true)

// Try to find any editable field
val editableNode = findFirstEditableNode(rootNode!!)
val editableNode = findFirstEditableNode(currentRootNode)

if (editableNode != null) {
Log.d(TAG, "Found editable node")
Expand Down Expand Up @@ -811,16 +828,9 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
// Refresh the root node
refreshRootNode()

// Check if root node is available
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing() // Continue queue if rootNode is null
return
}

// Try to find the node with the specified text
val node = findNodeByText(rootNode!!, buttonText)
val currentRootNode = currentRootNodeOrHandleMissing("find button", scheduleNext = true) ?: return
val node = findNodeByText(currentRootNode, buttonText)

if (node != null) {
Log.d(TAG, "Found node with text: $buttonText")
Expand Down Expand Up @@ -881,14 +891,8 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
showToast("Searching for button to long click with text: \"$buttonText\"", false)

refreshRootNode()
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing()
return
}

val node = findNodeByText(rootNode!!, buttonText)
val currentRootNode = currentRootNodeOrHandleMissing("find button", scheduleNext = true) ?: return
val node = findNodeByText(currentRootNode, buttonText)
if (node != null) {
Log.d(TAG, "Found node with text: $buttonText")
showToast("Button found: \"$buttonText\"", false)
Expand Down Expand Up @@ -918,14 +922,8 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
Log.d(TAG, "Finding and long clicking button with content description: $description")
showToast("Searching for button to long click with description: \"$description\"", false)

if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button by content description")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing()
return
}

val node = findNodeByContentDescription(rootNode!!, description)
val currentRootNode = currentRootNodeOrHandleMissing("find button by content description", scheduleNext = true) ?: return
val node = findNodeByContentDescription(currentRootNode, description)
if (node != null) {
Log.d(TAG, "Found node with content description: $description")
showToast("Button found with description: \"$description\"", false)
Expand Down Expand Up @@ -955,14 +953,8 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
Log.d(TAG, "Finding and long clicking button with ID: $id")
showToast("Searching for button to long click with ID: \"$id\"", false)

if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button by ID")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing()
return
}

val node = findNodeById(rootNode!!, id)
val currentRootNode = currentRootNodeOrHandleMissing("find button by ID", scheduleNext = true) ?: return
val node = findNodeById(currentRootNode, id)
if (node != null) {
Log.d(TAG, "Found node with ID: $id")
showToast("Button found with ID: \"$id\"", false)
Expand Down Expand Up @@ -993,16 +985,9 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
Log.d(TAG, "Finding and clicking button with content description: $description")
showToast("Searching for button with description: \"$description\"", false)

// Check if root node is available
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button by content description")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing() // Continue queue if rootNode is null
return
}

// Try to find the node with the specified content description
val node = findNodeByContentDescription(rootNode!!, description)
val currentRootNode = currentRootNodeOrHandleMissing("find button by content description", scheduleNext = true) ?: return
val node = findNodeByContentDescription(currentRootNode, description)

if (node != null) {
Log.d(TAG, "Found node with content description: $description")
Expand Down Expand Up @@ -1042,16 +1027,9 @@ class ScreenOperatorAccessibilityService : AccessibilityService() {
Log.d(TAG, "Finding and clicking button with ID: $id")
showToast("Searching for button with ID: \"$id\"", false)

// Check if root node is available
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot find button by ID")
showToast("Error: Root node is not available", true)
scheduleNextCommandProcessing() // Continue queue if rootNode is null
return
}

// Try to find the node with the specified ID
val node = findNodeById(rootNode!!, id)
val currentRootNode = currentRootNodeOrHandleMissing("find button by ID", scheduleNext = true) ?: return
val node = findNodeById(currentRootNode, id)

if (node != null) {
Log.d(TAG, "Found node with ID: $id")
Expand Down Expand Up @@ -1654,18 +1632,18 @@ private fun openAppUsingLaunchIntent(packageName: String, appName: String): Bool
// Refresh the root node to ensure we have the latest information
refreshRootNode()

// Check if root node is available
if (rootNode == null) {
Log.e(TAG, "Root node is null, cannot capture screen information")
return "No screen information available (root node is null)"
}

// Build a string with information about all interactive elements
val screenInfo = StringBuilder()
screenInfo.append("Screen elements:\n")

// Find all interactive elements
val elements = findAllInteractiveElements(rootNode!!)
val currentRootNode = currentRootNodeOrHandleMissing(
operation = "capture screen information",
scheduleNext = false,
showUserMessage = false
)
?: return "No screen information available (root node is null)"
val elements = findAllInteractiveElements(currentRootNode)

// Add information about each element
elements.forEachIndexed { index, element ->
Expand Down
25 changes: 7 additions & 18 deletions app/src/main/kotlin/com/google/ai/sample/feature/multimodal/PhotoReasoningScreen.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,12 @@ import androidx.compose.material3.Checkbox
import androidx.compose.material3.CheckboxDefaults
import androidx.compose.material3.DropdownMenu
import androidx.compose.material3.DropdownMenuItem
// Removed duplicate block:
// import androidx.compose.material3.AlertDialog
// import androidx.compose.material3.Button
// import androidx.compose.material3.ButtonDefaults
// import androidx.compose.material3.Card
// import androidx.compose.material3.CardDefaults
// import androidx.compose.material3.CircularProgressIndicator
// import androidx.compose.material3.Divider
// import androidx.compose.material3.Checkbox
// import androidx.compose.material3.CheckboxDefaults
// import androidx.compose.material3.DropdownMenu
// import androidx.compose.material3.DropdownMenuItem
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedTextField
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton // Existing, ensure it's not duplicated
import androidx.compose.material3.TextButton
import androidx.compose.material3.TextFieldDefaults
import androidx.compose.runtime.Composable
import androidx.compose.runtime.DisposableEffect
Expand Down Expand Up @@ -154,12 +142,14 @@ fun StopButton(onClick: () -> Unit) {
@Composable
internal fun PhotoReasoningRoute(
innerPadding: PaddingValues, // Füge Parameter hinzu
viewModelStoreOwner: androidx.lifecycle.ViewModelStoreOwner = androidx.lifecycle.viewmodel.compose.LocalViewModelStoreOwner.current!!
viewModelStoreOwner: androidx.lifecycle.ViewModelStoreOwner = checkNotNull(
androidx.lifecycle.viewmodel.compose.LocalViewModelStoreOwner.current
) { "ViewModelStoreOwner is required" }
) {
val context = LocalContext.current
val mainActivity = context as? MainActivity

// Scoped to MainActivity so it survives navigation, fixing duplicate init (Task 20)
// Scoped to MainActivity so it survives navigation and avoids duplicate initialization.
val owner = mainActivity ?: viewModelStoreOwner
val viewModel: PhotoReasoningViewModel = androidx.lifecycle.viewmodel.compose.viewModel(
viewModelStoreOwner = owner,
Expand All @@ -179,7 +169,6 @@ internal fun PhotoReasoningRoute(

// Hoisted: var showNotificationRationaleDialog by rememberSaveable { mutableStateOf(false) }
// This state will now be managed in PhotoReasoningRoute and passed down.
// var showNotificationRationaleDialogStateInRoute by rememberSaveable { mutableStateOf(false) } // Removed


val coroutineScope = rememberCoroutineScope()
Expand Down Expand Up @@ -811,7 +800,7 @@ fun DatabaseListPopup(
)

if (entryToConfirmOverwrite != null) {
val (existingEntry, newEntry) = entryToConfirmOverwrite!!
val (existingEntry, newEntry) = checkNotNull(entryToConfirmOverwrite)
OverwriteConfirmationDialog(
entryTitle = newEntry.title,
onConfirm = {
Expand Down Expand Up @@ -1393,7 +1382,7 @@ fun VerticalScrollbar(
}

if (scrollbarState != null) {
val (offset, height) = scrollbarState!!
val (offset, height) = checkNotNull(scrollbarState)
Canvas(modifier = modifier.width(4.dp)) {
drawRoundRect(
color = Color.Gray,
Expand Down
Loading