From 1fefde2cd7d93c8ac721025db1f40fd6d4b7eed4 Mon Sep 17 00:00:00 2001 From: JordanTheJet Date: Fri, 20 Feb 2026 16:37:50 -0500 Subject: [PATCH] Add notification actions: reply, explain screen, approve/deny - New NotificationActionReceiver with 4 actions (reply via RemoteInput, screenshot+explain, approve all, deny all) using @EntryPoint for Hilt - Rewrite CellClawService notification with live state observation via combine(agentLoop.state, approvalQueue.requests), dynamic Pause/Resume toggle, and conditional approve/deny buttons when approvals pending - Register NotificationActionReceiver in AndroidManifest.xml Co-Authored-By: Claude Opus 4.6 --- app/src/main/AndroidManifest.xml | 11 ++ .../com/cellclaw/service/CellClawService.kt | 115 +++++++++++++-- .../receivers/NotificationActionReceiver.kt | 135 ++++++++++++++++++ 3 files changed, 246 insertions(+), 15 deletions(-) create mode 100644 app/src/main/kotlin/com/cellclaw/service/receivers/NotificationActionReceiver.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 2d782b4..3180579 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -160,6 +160,17 @@ + + + + + + + + + { - startForeground(NOTIFICATION_ID, buildNotification("CellClaw is running")) + startForeground(NOTIFICATION_ID, buildNotification("CellClaw is running", AgentState.IDLE, false)) agentLoop.loadHistory() + observeState() } ACTION_STOP -> { agentLoop.stop() + serviceScope?.cancel() + serviceScope = null stopForeground(STOP_FOREGROUND_REMOVE) stopSelf() } ACTION_PAUSE -> { agentLoop.pause() - updateNotification("CellClaw is paused") } ACTION_RESUME -> { agentLoop.resume() - updateNotification("CellClaw is running") } } return START_STICKY } override fun onDestroy() { + serviceScope?.cancel() + serviceScope = null releaseWakeLock() super.onDestroy() } - private fun buildNotification(text: String): Notification { + private fun observeState() { + serviceScope?.cancel() + serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + serviceScope?.launch { + combine(agentLoop.state, approvalQueue.requests) { state, requests -> + Pair(state, requests) + }.collect { (state, requests) -> + val hasPending = requests.isNotEmpty() + val text = when (state) { + AgentState.IDLE -> "Idle" + AgentState.THINKING -> "Thinking..." + AgentState.EXECUTING_TOOLS -> "Executing tools..." + AgentState.WAITING_APPROVAL -> "Waiting for approval (${requests.size} pending)" + AgentState.PAUSED -> "Paused" + AgentState.ERROR -> "Error occurred" + } + val notification = buildNotification(text, state, hasPending) + val manager = getSystemService(NOTIFICATION_SERVICE) as android.app.NotificationManager + manager.notify(NOTIFICATION_ID, notification) + } + } + } + + private fun buildNotification(text: String, state: AgentState, hasPendingApprovals: Boolean): Notification { val openIntent = PendingIntent.getActivity( this, 0, Intent(this, MainActivity::class.java), @@ -69,28 +103,79 @@ class CellClawService : Service() { PendingIntent.FLAG_IMMUTABLE ) - val pauseIntent = PendingIntent.getService( - this, 2, - Intent(this, CellClawService::class.java).apply { action = ACTION_PAUSE }, + // Pause/Resume toggle + val isPaused = state == AgentState.PAUSED + val toggleAction = if (isPaused) { + PendingIntent.getService( + this, 2, + Intent(this, CellClawService::class.java).apply { action = ACTION_RESUME }, + PendingIntent.FLAG_IMMUTABLE + ) + } else { + PendingIntent.getService( + this, 2, + Intent(this, CellClawService::class.java).apply { action = ACTION_PAUSE }, + PendingIntent.FLAG_IMMUTABLE + ) + } + + // Reply action via RemoteInput (shows as text field, not a button) + val replyIntent = PendingIntent.getBroadcast( + this, 3, + Intent(this, NotificationActionReceiver::class.java).apply { + action = NotificationActionReceiver.ACTION_REPLY + }, + PendingIntent.FLAG_MUTABLE or PendingIntent.FLAG_UPDATE_CURRENT + ) + val remoteInput = RemoteInput.Builder(NotificationActionReceiver.KEY_REPLY) + .setLabel("Ask CellClaw...") + .build() + val replyAction = NotificationCompat.Action.Builder(0, "Reply", replyIntent) + .addRemoteInput(remoteInput) + .build() + + // Explain Screen action + val explainIntent = PendingIntent.getBroadcast( + this, 4, + Intent(this, NotificationActionReceiver::class.java).apply { + action = NotificationActionReceiver.ACTION_SCREENSHOT_EXPLAIN + }, PendingIntent.FLAG_IMMUTABLE ) - return NotificationCompat.Builder(this, CellClawApp.CHANNEL_SERVICE) + val builder = NotificationCompat.Builder(this, CellClawApp.CHANNEL_SERVICE) .setContentTitle("CellClaw") .setContentText(text) .setSmallIcon(R.drawable.ic_notification) .setContentIntent(openIntent) .addAction(0, "Stop", stopIntent) - .addAction(0, "Pause", pauseIntent) + .addAction(0, if (isPaused) "Resume" else "Pause", toggleAction) + .addAction(0, "Explain Screen", explainIntent) + .addAction(replyAction) .setOngoing(true) .setSilent(true) - .build() - } - private fun updateNotification(text: String) { - val notification = buildNotification(text) - val manager = getSystemService(NOTIFICATION_SERVICE) as android.app.NotificationManager - manager.notify(NOTIFICATION_ID, notification) + // Approve/Deny actions in expanded area when approvals are pending + if (hasPendingApprovals) { + val approveIntent = PendingIntent.getBroadcast( + this, 5, + Intent(this, NotificationActionReceiver::class.java).apply { + action = NotificationActionReceiver.ACTION_APPROVE_ALL + }, + PendingIntent.FLAG_IMMUTABLE + ) + val denyIntent = PendingIntent.getBroadcast( + this, 6, + Intent(this, NotificationActionReceiver::class.java).apply { + action = NotificationActionReceiver.ACTION_DENY_ALL + }, + PendingIntent.FLAG_IMMUTABLE + ) + builder.addAction(0, "Approve All", approveIntent) + builder.addAction(0, "Deny All", denyIntent) + } + + return builder.build() } private fun acquireWakeLock() { diff --git a/app/src/main/kotlin/com/cellclaw/service/receivers/NotificationActionReceiver.kt b/app/src/main/kotlin/com/cellclaw/service/receivers/NotificationActionReceiver.kt new file mode 100644 index 0000000..2f7d56b --- /dev/null +++ b/app/src/main/kotlin/com/cellclaw/service/receivers/NotificationActionReceiver.kt @@ -0,0 +1,135 @@ +package com.cellclaw.service.receivers + +import android.app.NotificationManager +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.util.Log +import androidx.core.app.NotificationCompat +import androidx.core.app.RemoteInput +import com.cellclaw.CellClawApp +import com.cellclaw.R +import com.cellclaw.agent.AgentLoop +import com.cellclaw.approval.ApprovalQueue +import com.cellclaw.approval.ApprovalResult +import com.cellclaw.tools.ScreenCaptureTool +import com.cellclaw.tools.VisionAnalyzeTool +import dagger.hilt.EntryPoint +import dagger.hilt.InstallIn +import dagger.hilt.android.EntryPointAccessors +import dagger.hilt.components.SingletonComponent +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put + +class NotificationActionReceiver : BroadcastReceiver() { + + @EntryPoint + @InstallIn(SingletonComponent::class) + interface ReceiverEntryPoint { + fun agentLoop(): AgentLoop + fun approvalQueue(): ApprovalQueue + fun screenCaptureTool(): ScreenCaptureTool + fun visionAnalyzeTool(): VisionAnalyzeTool + } + + override fun onReceive(context: Context, intent: Intent) { + val entryPoint = EntryPointAccessors.fromApplication( + context.applicationContext, ReceiverEntryPoint::class.java + ) + + when (intent.action) { + ACTION_REPLY -> handleReply(context, intent, entryPoint) + ACTION_SCREENSHOT_EXPLAIN -> handleScreenshotExplain(context, entryPoint) + ACTION_APPROVE_ALL -> { + entryPoint.approvalQueue().respondAll(ApprovalResult.APPROVED) + Log.d(TAG, "Approved all pending requests") + } + ACTION_DENY_ALL -> { + entryPoint.approvalQueue().respondAll(ApprovalResult.DENIED) + Log.d(TAG, "Denied all pending requests") + } + } + } + + private fun handleReply(context: Context, intent: Intent, entryPoint: ReceiverEntryPoint) { + val remoteInput = RemoteInput.getResultsFromIntent(intent) + val text = remoteInput?.getCharSequence(KEY_REPLY)?.toString() + if (text.isNullOrBlank()) return + + Log.d(TAG, "Reply received: $text") + entryPoint.agentLoop().submitMessage(text) + } + + private fun handleScreenshotExplain(context: Context, entryPoint: ReceiverEntryPoint) { + val pendingResult = goAsync() + CoroutineScope(Dispatchers.IO).launch { + try { + val screenCapture = entryPoint.screenCaptureTool() + val visionAnalyze = entryPoint.visionAnalyzeTool() + + val captureResult = screenCapture.execute( + buildJsonObject { put("include_base64", false) } + ) + if (!captureResult.success) { + postResultNotification(context, "Screenshot failed: ${captureResult.error}") + return@launch + } + + val filePath = captureResult.data?.let { data -> + (data as? kotlinx.serialization.json.JsonObject)?.get("file_path") + ?.let { (it as? kotlinx.serialization.json.JsonPrimitive)?.content } + } + if (filePath == null) { + postResultNotification(context, "Screenshot failed: no file path") + return@launch + } + + val analyzeResult = visionAnalyze.execute(buildJsonObject { + put("file_path", filePath) + put("question", "Describe what you see on the screen. What app is open? What is the user looking at?") + }) + + if (analyzeResult.success) { + val analysis = analyzeResult.data?.let { data -> + (data as? kotlinx.serialization.json.JsonObject)?.get("analysis") + ?.let { (it as? kotlinx.serialization.json.JsonPrimitive)?.content } + } ?: "Analysis complete" + postResultNotification(context, analysis) + } else { + postResultNotification(context, "Analysis failed: ${analyzeResult.error}") + } + } catch (e: Exception) { + Log.e(TAG, "Screenshot explain failed: ${e.message}") + postResultNotification(context, "Error: ${e.message}") + } finally { + pendingResult.finish() + } + } + } + + private fun postResultNotification(context: Context, text: String) { + val notification = NotificationCompat.Builder(context, CellClawApp.CHANNEL_ALERTS) + .setContentTitle("Screen Analysis") + .setContentText(text) + .setStyle(NotificationCompat.BigTextStyle().bigText(text)) + .setSmallIcon(R.drawable.ic_notification) + .setAutoCancel(true) + .build() + + val manager = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.notify(EXPLAIN_NOTIFICATION_ID, notification) + } + + companion object { + private const val TAG = "NotifActionReceiver" + const val ACTION_REPLY = "com.cellclaw.NOTIFICATION_REPLY" + const val ACTION_SCREENSHOT_EXPLAIN = "com.cellclaw.SCREENSHOT_EXPLAIN" + const val ACTION_APPROVE_ALL = "com.cellclaw.APPROVE_ALL" + const val ACTION_DENY_ALL = "com.cellclaw.DENY_ALL" + const val KEY_REPLY = "key_reply_text" + const val EXPLAIN_NOTIFICATION_ID = 100 + } +}