From 2ecfa708e6c16bb8d62a4e3102e9b1a993021a24 Mon Sep 17 00:00:00 2001 From: Mathieu B Date: Sun, 4 Dec 2022 12:31:46 +0100 Subject: [PATCH] Re-arranged pages, fixed lag issue with socket.on and working sockets response. --- app/build.gradle | 1 - .../broillet/jarvis/android/MainActivity.kt | 41 ++-- .../jarvis/android/audio/AudioRecorder.kt | 3 - .../broillet/jarvis/android/nav/Navigation.kt | 20 +- .../ch/broillet/jarvis/android/nav/Screen.kt | 2 +- .../broillet/jarvis/android/pages/MainPage.kt | 176 ++++++++++-------- .../jarvis/android/pages/PermissionsPage.kt | 60 ++++-- .../jarvis/android/utils/PermissionsUtils.kt | 49 ----- .../jarvis/android/utils/RequestsUtils.kt | 1 - .../jarvis/android/utils/SocketHandler.kt | 11 ++ .../ch/broillet/jarvis/android/utils/Utils.kt | 1 + 11 files changed, 187 insertions(+), 178 deletions(-) delete mode 100644 app/src/main/java/ch/broillet/jarvis/android/utils/PermissionsUtils.kt diff --git a/app/build.gradle b/app/build.gradle index b5fc77c..70c82f4 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -71,7 +71,6 @@ dependencies { implementation 'com.github.squti:Android-Wave-Recorder:1.7.0' implementation("com.squareup.okhttp3:okhttp:4.9.3") implementation 'com.google.code.gson:gson:2.8.9' - implementation ('io.socket:socket.io-client:2.0.0') { exclude group: 'org.json', module: 'json' } diff --git a/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt b/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt index b4ac8cc..352d921 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt @@ -1,35 +1,53 @@ package ch.broillet.jarvis.android import android.os.Bundle +import android.provider.Settings import android.view.WindowManager import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface -import androidx.compose.runtime.Composable import androidx.compose.ui.Modifier -import androidx.compose.ui.tooling.preview.Preview import androidx.core.view.WindowCompat +import ch.broillet.jarvis.android.chat.ConversationUiState +import ch.broillet.jarvis.android.chat.Message import ch.broillet.jarvis.android.nav.Navigation import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme import ch.broillet.jarvis.android.utils.SocketHandler -import java.util.* class MainActivity : ComponentActivity() { - var uniqueID = UUID.randomUUID().toString() + + + // Default uiState with welcome message only + var uiState = ConversationUiState(listOf()) override fun onCreate(savedInstanceState: Bundle?) { + uiState = ConversationUiState( + listOf( + Message( + true, + resources.getString(R.string.demo_message_1) + ) + ) + ) // The following lines connects the Android app to the server. SocketHandler.setSocket() SocketHandler.establishConnection() - SocketHandler.joinRoom(uniqueID) + SocketHandler.joinRoom( + Settings.Secure.getString( + applicationContext.contentResolver, + Settings.Secure.ANDROID_ID + ) + ) + + SocketHandler.getSocket() + .on("message_from_jarvis") { SocketHandler.messageFromJarvis(it, uiState) } super.onCreate(savedInstanceState) - setContent { JarvisclientappTheme { @@ -44,18 +62,9 @@ class MainActivity : ComponentActivity() { Surface( modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background ) { - DefaultPreview() + Navigation(uiState) } } } } -} - - -@Preview(showBackground = true) -@Composable -fun DefaultPreview() { - JarvisclientappTheme { - Navigation() - } } \ No newline at end of file diff --git a/app/src/main/java/ch/broillet/jarvis/android/audio/AudioRecorder.kt b/app/src/main/java/ch/broillet/jarvis/android/audio/AudioRecorder.kt index 8e81137..6e0c503 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/audio/AudioRecorder.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/audio/AudioRecorder.kt @@ -10,15 +10,12 @@ class AudioRecorder( var waveRecorder: WaveRecorder ) { - fun startRecording() { waveRecorder.startRecording() } fun stopRecording() { waveRecorder.stopRecording() - - } fun getOutputFile(): File { diff --git a/app/src/main/java/ch/broillet/jarvis/android/nav/Navigation.kt b/app/src/main/java/ch/broillet/jarvis/android/nav/Navigation.kt index c0c3b27..d2e2b94 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/nav/Navigation.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/nav/Navigation.kt @@ -2,15 +2,12 @@ package ch.broillet.jarvis.android.nav import androidx.compose.runtime.Composable import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.stringResource import androidx.navigation.NavController import androidx.navigation.compose.NavHost import androidx.navigation.compose.composable import androidx.navigation.compose.rememberNavController -import ch.broillet.jarvis.android.R import ch.broillet.jarvis.android.audio.getAudioRecorder import ch.broillet.jarvis.android.chat.ConversationUiState -import ch.broillet.jarvis.android.chat.Message import ch.broillet.jarvis.android.pages.DisplayMainPage import ch.broillet.jarvis.android.pages.DisplayPermissionsPage import ch.broillet.jarvis.android.pages.DisplaySettingsPage @@ -19,12 +16,11 @@ import com.google.accompanist.permissions.rememberMultiplePermissionsState @OptIn(ExperimentalPermissionsApi::class) @Composable -fun Navigation() { +fun Navigation(uiState: ConversationUiState) { val navController = rememberNavController() val permissions = listOf( android.Manifest.permission.RECORD_AUDIO, - //android.Manifest.permission.WRITE_EXTERNAL_STORAGE ) val permissionState = rememberMultiplePermissionsState(permissions) @@ -34,7 +30,7 @@ fun Navigation() { startDestination = if (permissionState.allPermissionsGranted) Screen.MainScreen.route else Screen.PermissionsScreen.route ) { composable(route = Screen.MainScreen.route) { - MainScreen(navController = navController) + MainScreen(navController = navController, uiState) } composable(route = Screen.SettingsScreen.route) { SettingsScreen(navController = navController) @@ -47,17 +43,11 @@ fun Navigation() { } @Composable -fun MainScreen(navController: NavController) { +fun MainScreen(navController: NavController, uiState: ConversationUiState) { + //TODO: change so that it doesn't reset when going to settings menu DisplayMainPage( navController, - ConversationUiState( - listOf( - Message( - true, - stringResource(id = R.string.demo_message_1) - ) - ) - ), + uiState, getAudioRecorder(LocalContext.current) ) } diff --git a/app/src/main/java/ch/broillet/jarvis/android/nav/Screen.kt b/app/src/main/java/ch/broillet/jarvis/android/nav/Screen.kt index 02826a2..c5cfe7f 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/nav/Screen.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/nav/Screen.kt @@ -1,6 +1,6 @@ package ch.broillet.jarvis.android.nav -sealed class Screen(val route: String){ +sealed class Screen(val route: String) { object MainScreen : Screen("main_screen") object SettingsScreen : Screen("settings_screen") object PermissionsScreen : Screen("permissions_screen") diff --git a/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt b/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt index 72b3db9..9906152 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt @@ -1,6 +1,6 @@ package ch.broillet.jarvis.android.pages -import android.os.Looper +import android.provider.Settings.Secure import androidx.compose.foundation.layout.* import androidx.compose.material.icons.Icons import androidx.compose.material.icons.outlined.Edit @@ -18,7 +18,6 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.navigation.NavController import androidx.navigation.compose.rememberNavController -import ch.broillet.jarvis.android.MainActivity import ch.broillet.jarvis.android.R import ch.broillet.jarvis.android.audio.AudioRecorder import ch.broillet.jarvis.android.chat.ConversationUiState @@ -31,12 +30,50 @@ import ch.broillet.jarvis.android.utils.* import com.github.squti.androidwaverecorder.RecorderState import com.github.squti.androidwaverecorder.WaveRecorder import org.json.JSONObject +import java.io.File import kotlin.concurrent.thread +@Composable +fun DisplayMainPage( + navController: NavController, + uiState: ConversationUiState, + audioRecorder: AudioRecorder +) { + + //We create a main box with basic padding to avoid having stuff too close to every side. + DefaultBox { + + // This column regroup the base and all the conversations (everything except the footer) + Column(Modifier.padding(bottom = 80.dp)) { + + MainBase(navController) + + Messages( + messages = uiState.messages, + modifier = Modifier.weight(1f) + ) + } + + // Finally we add the footer to the bottom center of the main box + Column( + Modifier + .align(Alignment.BottomCenter) + .padding(bottom = 40.dp) + ) { + + RecordingFooterButton( + audioRecorder = audioRecorder, + navController = navController, + uiState = uiState + ) + } + } +} + //Draws the base of the main activity, that includes the 3-dots menu and the "hi text". @Composable -fun Base(navController: NavController) { +fun MainBase(navController: NavController) { Column( Modifier @@ -103,7 +140,38 @@ fun DropDownSettingsMenu(navController: NavController) { } @Composable -fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: Boolean) { +fun RecordingFooterButton( + audioRecorder: AudioRecorder, + navController: NavController, + uiState: ConversationUiState +) { + + var isListening: Boolean by remember { mutableStateOf(false) } + var isProcessing: Boolean by remember { mutableStateOf(false) } + + // Add a listener for the waveRecorder to record when isListening is true and then process the audio when done listening + audioRecorder.waveRecorder.onStateChangeListener = { + when (it) { + RecorderState.RECORDING -> isListening = true + RecorderState.STOP -> { + thread { + isListening = false + isProcessing = true + + processMessage( + processAudio(audioRecorder.getOutputFile()), + navController, + uiState + ) + + isProcessing = false + audioRecorder.getOutputFile().delete() + } + } + else -> {} + } + } + //We create a row that we align to the bottom center of the parent box Row( Modifier @@ -113,8 +181,11 @@ fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: B ) { //Microphone floating button to manually start/stop listening - FloatingActionButton(onClick = onClick, modifier = Modifier.size(70.dp)) { - if (isRecording) { + FloatingActionButton( + onClick = { if (isListening) audioRecorder.stopRecording() else audioRecorder.startRecording() }, + modifier = Modifier.size(70.dp) + ) { + if (isListening) { DotsTyping(7.dp, 3, 300, MaterialTheme.colorScheme.secondary, 2.dp) } else { if (isProcessing) { @@ -130,84 +201,27 @@ fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: B } } +fun processAudio(audioFile: File): String { + val json = JSONObject(getTextFromAudio(audioFile)) -@Composable -fun DisplayMainPage( - navController: NavController, - uiState: ConversationUiState, - audioRecorder: AudioRecorder -) { - - - //We create a main box with basic padding to avoid having stuff too close to every side. - DefaultBox { - - // This column regroup the base and all the conversations (everything except the footer) - Column(Modifier.padding(bottom = 80.dp)) { - - Base(navController) - - Messages( - messages = uiState.messages, - modifier = Modifier.weight(1f) - ) - } - - // Finally we add the footer to the bottom center of the main box - Column( - Modifier - .align(Alignment.BottomCenter) - .padding(bottom = 40.dp) - ) { - var listening: Boolean by remember { mutableStateOf(false) } - var processing: Boolean by remember { mutableStateOf(false) } - - SocketHandler.getSocket().on("message_from_jarvis") { args -> - if (args[0] != null) { - uiState.addMessage(Message(true, args.toString())) - } - } - audioRecorder.waveRecorder.onStateChangeListener = { - when (it) { - RecorderState.RECORDING -> listening = true - RecorderState.STOP -> { - listening = false - processing = true - - SocketHandler.processMessage("test", MainActivity().uniqueID) - - thread { - val requestOutput = getTextFromAudio(audioRecorder.getOutputFile()) - - /*val temp = JSONObject() - temp.put("data", "salut je suis bob") - val requestOutput = temp.toString()*/ - - processing = false - - val json = JSONObject(requestOutput) - val sent = json.getString("data") - - uiState.addMessage(Message(false, sent)) - - // Thread.sleep(1000) - // uiState.addMessage(Message(true, json.getString("answer"))) - audioRecorder.getOutputFile().delete() - } - } - else -> {} - } - } - - StartRecordingFAB( - onClick = { if (listening) audioRecorder.stopRecording() else audioRecorder.startRecording() }, - isRecording = listening, - isProcessing = processing - ) - } - } + return json.getString("data") } +fun processMessage(text: String, navController: NavController, uiState: ConversationUiState) { + navController.context.mainExecutor.execute { + SocketHandler.processMessage( + text, + Secure.getString( + navController.context.contentResolver, + Secure.ANDROID_ID + ) + ) + } + + uiState.addMessage(Message(false, text)) +} + + @Preview(showBackground = true) @Composable fun MainPagePreview() { diff --git a/app/src/main/java/ch/broillet/jarvis/android/pages/PermissionsPage.kt b/app/src/main/java/ch/broillet/jarvis/android/pages/PermissionsPage.kt index 79acd13..9441c86 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/pages/PermissionsPage.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/pages/PermissionsPage.kt @@ -2,11 +2,12 @@ package ch.broillet.jarvis.android.pages import android.Manifest import androidx.compose.foundation.layout.* -import androidx.compose.material3.Icon -import androidx.compose.material3.Text +import androidx.compose.material3.* import androidx.compose.runtime.Composable import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.tooling.preview.Preview @@ -18,7 +19,10 @@ import ch.broillet.jarvis.android.R import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme import ch.broillet.jarvis.android.ui.theme.productSansFont import ch.broillet.jarvis.android.utils.DefaultBox -import ch.broillet.jarvis.android.utils.requestPermissionButton +import ch.broillet.jarvis.android.utils.openAppSettings +import com.google.accompanist.permissions.ExperimentalPermissionsApi +import com.google.accompanist.permissions.PermissionRequired +import com.google.accompanist.permissions.rememberPermissionState @Composable fun DisplayPermissionsPage(navController: NavController) { @@ -54,19 +58,13 @@ fun PermissionsBase() { modifier = Modifier.padding(top = 10.dp) ) + // List of required permissions to display PermissionRow( R.drawable.ic_baseline_mic_24, stringResource(id = R.string.permission_microphone), stringResource(id = R.string.permission_microphone_description), Manifest.permission.RECORD_AUDIO ) - - /*PermissionRow( - R.drawable.ic_baseline_folder_open_24, - stringResource(id = R.string.permission_files), - stringResource(id = R.string.permission_files_description), - Manifest.permission.WRITE_EXTERNAL_STORAGE - )*/ } } @@ -106,10 +104,50 @@ fun PermissionRow( } } - requestPermissionButton(permission = permission) + // Remove preview error (only in IDE, has no impact on app) + if (!LocalInspectionMode.current) { + PermissionRequestButton(permission = permission) + } } } + +@OptIn(ExperimentalPermissionsApi::class) +@Composable +fun PermissionRequestButton(permission: String) { + val permissionState = rememberPermissionState(permission = permission) + val context = LocalContext.current + + PermissionRequired( + permissionState = permissionState, + permissionNotGrantedContent = { + Button(onClick = { + permissionState.launchPermissionRequest() + }) { + Text(text = stringResource(id = R.string.permissions_page_grant_permission)) + } + + }, + permissionNotAvailableContent = { + Button( + onClick = { openAppSettings(context) }, + colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colorScheme.error) + ) { + Text(text = stringResource(id = R.string.permissions_page_permission_denied)) + Icon( + painter = painterResource(id = R.drawable.ic_outline_settings_24), + contentDescription = "app settings icon" + ) + } + }, + content = { + Button(onClick = {}, enabled = false) { + Text(text = stringResource(id = R.string.permission_granted)) + } + } + ) +} + @Preview(showBackground = true) @Composable fun PermissionsPagePreview() { diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/PermissionsUtils.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/PermissionsUtils.kt deleted file mode 100644 index 01c41f6..0000000 --- a/app/src/main/java/ch/broillet/jarvis/android/utils/PermissionsUtils.kt +++ /dev/null @@ -1,49 +0,0 @@ -package ch.broillet.jarvis.android.utils - -import androidx.compose.material.* -import androidx.compose.material3.* -import androidx.compose.runtime.Composable -import androidx.compose.ui.platform.LocalContext -import androidx.compose.ui.res.painterResource -import androidx.compose.ui.res.stringResource -import com.google.accompanist.permissions.ExperimentalPermissionsApi -import com.google.accompanist.permissions.PermissionRequired -import com.google.accompanist.permissions.rememberPermissionState -import ch.broillet.jarvis.android.R - - -@OptIn(ExperimentalPermissionsApi::class) -@Composable -fun requestPermissionButton(permission: String) { - val permissionState = rememberPermissionState(permission = permission) - val context = LocalContext.current - - PermissionRequired( - permissionState = permissionState, - permissionNotGrantedContent = { - Button(onClick = { - permissionState.launchPermissionRequest() - }) { - Text(text = stringResource(id = R.string.permissions_page_grant_permission)) - } - - }, - permissionNotAvailableContent = { - Button( - onClick = { openAppSettings(context) }, - colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colorScheme.error) - ) { - Text(text = stringResource(id = R.string.permissions_page_permission_denied)) - Icon( - painter = painterResource(id = R.drawable.ic_outline_settings_24), - contentDescription = "app settings icon" - ) - } - }, - content = { - Button(onClick = {}, enabled = false) { - Text(text = stringResource(id = R.string.permission_granted)) - } - } - ) -} \ No newline at end of file diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt index 22a17bc..39274ba 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt @@ -6,7 +6,6 @@ import okhttp3.Request import okhttp3.RequestBody.Companion.asRequestBody import java.io.File import java.io.IOException -import java.net.Socket fun getTextFromAudio(file: File): String { diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt index f88cd4f..5bf4399 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt @@ -1,5 +1,7 @@ package ch.broillet.jarvis.android.utils +import ch.broillet.jarvis.android.chat.ConversationUiState +import ch.broillet.jarvis.android.chat.Message import io.socket.client.IO import io.socket.client.Socket import org.json.JSONObject @@ -46,4 +48,13 @@ object SocketHandler { body.put("uuid", uuid) getSocket().emit("join", body.toString()) } + + + fun messageFromJarvis(data: Array, uiState: ConversationUiState) { + if (data[0].toString().contains("data")) { + val result: JSONObject = data[0] as JSONObject + uiState.addMessage(Message(true, result.getString("data"))) + } + + } } \ No newline at end of file diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/Utils.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/Utils.kt index 2ff7c62..108629a 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/utils/Utils.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/utils/Utils.kt @@ -26,6 +26,7 @@ fun DefaultBox( } +// Open the application in the system's settings to (i.e) manually give permission fun openAppSettings(context: Context) { context.startActivity( Intent(