mathieu/jarvis-client-android
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Re-arranged pages, fixed lag issue with socket.on and working sockets response.

Browse Source
This commit is contained in:
Mathieu B 2022-12-04 12:31:46 +01:00
parent 8c50b26b71
commit 2ecfa708e6
11 changed files with 187 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/build.gradle
View File

@ -71,7 +71,6 @@ dependencies {
implementation 'com.github.squti:Android-Wave-Recorder:1.7.0'
implementation("com.squareup.okhttp3:okhttp:4.9.3")
implementation 'com.google.code.gson:gson:2.8.9'
implementation ('io.socket:socket.io-client:2.0.0') {
exclude group: 'org.json', module: 'json'
}

41
app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt
View File

@ -1,35 +1,53 @@
package ch.broillet.jarvis.android
import android.os.Bundle
import android.provider.Settings
import android.view.WindowManager
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.runtime.Composable
import androidx.compose.ui.Modifier
import androidx.compose.ui.tooling.preview.Preview
import androidx.core.view.WindowCompat
import ch.broillet.jarvis.android.chat.ConversationUiState
import ch.broillet.jarvis.android.chat.Message
import ch.broillet.jarvis.android.nav.Navigation
import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme
import ch.broillet.jarvis.android.utils.SocketHandler
import java.util.*
class MainActivity : ComponentActivity() {
var uniqueID = UUID.randomUUID().toString()
// Default uiState with welcome message only
var uiState = ConversationUiState(listOf())
override fun onCreate(savedInstanceState: Bundle?) {
uiState = ConversationUiState(
listOf(
Message(
true,
resources.getString(R.string.demo_message_1)
)
)
)
// The following lines connects the Android app to the server.
SocketHandler.setSocket()
SocketHandler.establishConnection()
SocketHandler.joinRoom(uniqueID)
SocketHandler.joinRoom(
Settings.Secure.getString(
applicationContext.contentResolver,
Settings.Secure.ANDROID_ID
)
)
SocketHandler.getSocket()
.on("message_from_jarvis") { SocketHandler.messageFromJarvis(it, uiState) }
super.onCreate(savedInstanceState)
setContent {
JarvisclientappTheme {
@ -44,18 +62,9 @@ class MainActivity : ComponentActivity() {
Surface(
modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background
) {
DefaultPreview()
Navigation(uiState)
}
}
}
}
}
@Preview(showBackground = true)
@Composable
fun DefaultPreview() {
JarvisclientappTheme {
Navigation()
}
}

3
app/src/main/java/ch/broillet/jarvis/android/audio/AudioRecorder.kt
View File

@ -10,15 +10,12 @@ class AudioRecorder(
var waveRecorder: WaveRecorder
) {
fun startRecording() {
waveRecorder.startRecording()
}
fun stopRecording() {
waveRecorder.stopRecording()
}
fun getOutputFile(): File {

20
app/src/main/java/ch/broillet/jarvis/android/nav/Navigation.kt
View File

@ -2,15 +2,12 @@ package ch.broillet.jarvis.android.nav
import androidx.compose.runtime.Composable
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.stringResource
import androidx.navigation.NavController
import androidx.navigation.compose.NavHost
import androidx.navigation.compose.composable
import androidx.navigation.compose.rememberNavController
import ch.broillet.jarvis.android.R
import ch.broillet.jarvis.android.audio.getAudioRecorder
import ch.broillet.jarvis.android.chat.ConversationUiState
import ch.broillet.jarvis.android.chat.Message
import ch.broillet.jarvis.android.pages.DisplayMainPage
import ch.broillet.jarvis.android.pages.DisplayPermissionsPage
import ch.broillet.jarvis.android.pages.DisplaySettingsPage
@ -19,12 +16,11 @@ import com.google.accompanist.permissions.rememberMultiplePermissionsState
@OptIn(ExperimentalPermissionsApi::class)
@Composable
fun Navigation() {
fun Navigation(uiState: ConversationUiState) {
val navController = rememberNavController()
val permissions = listOf(
android.Manifest.permission.RECORD_AUDIO,
//android.Manifest.permission.WRITE_EXTERNAL_STORAGE
)
val permissionState = rememberMultiplePermissionsState(permissions)
@ -34,7 +30,7 @@ fun Navigation() {
startDestination = if (permissionState.allPermissionsGranted) Screen.MainScreen.route else Screen.PermissionsScreen.route
) {
composable(route = Screen.MainScreen.route) {
MainScreen(navController = navController)
MainScreen(navController = navController, uiState)
}
composable(route = Screen.SettingsScreen.route) {
SettingsScreen(navController = navController)
@ -47,17 +43,11 @@ fun Navigation() {
}
@Composable
fun MainScreen(navController: NavController) {
fun MainScreen(navController: NavController, uiState: ConversationUiState) {
//TODO: change so that it doesn't reset when going to settings menu
DisplayMainPage(
navController,
ConversationUiState(
listOf(
Message(
true,
stringResource(id = R.string.demo_message_1)
)
)
),
uiState,
getAudioRecorder(LocalContext.current)
)
}

2
app/src/main/java/ch/broillet/jarvis/android/nav/Screen.kt
View File

@ -1,6 +1,6 @@
package ch.broillet.jarvis.android.nav
sealed class Screen(val route: String){
sealed class Screen(val route: String) {
object MainScreen : Screen("main_screen")
object SettingsScreen : Screen("settings_screen")
object PermissionsScreen : Screen("permissions_screen")

176
app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt
View File

@ -1,6 +1,6 @@
package ch.broillet.jarvis.android.pages
import android.os.Looper
import android.provider.Settings.Secure
import androidx.compose.foundation.layout.*
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.outlined.Edit
@ -18,7 +18,6 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.navigation.NavController
import androidx.navigation.compose.rememberNavController
import ch.broillet.jarvis.android.MainActivity
import ch.broillet.jarvis.android.R
import ch.broillet.jarvis.android.audio.AudioRecorder
import ch.broillet.jarvis.android.chat.ConversationUiState
@ -31,12 +30,50 @@ import ch.broillet.jarvis.android.utils.*
import com.github.squti.androidwaverecorder.RecorderState
import com.github.squti.androidwaverecorder.WaveRecorder
import org.json.JSONObject
import java.io.File
import kotlin.concurrent.thread
@Composable
fun DisplayMainPage(
navController: NavController,
uiState: ConversationUiState,
audioRecorder: AudioRecorder
) {
//We create a main box with basic padding to avoid having stuff too close to every side.
DefaultBox {
// This column regroup the base and all the conversations (everything except the footer)
Column(Modifier.padding(bottom = 80.dp)) {
MainBase(navController)
Messages(
messages = uiState.messages,
modifier = Modifier.weight(1f)
)
}
// Finally we add the footer to the bottom center of the main box
Column(
Modifier
.align(Alignment.BottomCenter)
.padding(bottom = 40.dp)
) {
RecordingFooterButton(
audioRecorder = audioRecorder,
navController = navController,
uiState = uiState
)
}
}
}
//Draws the base of the main activity, that includes the 3-dots menu and the "hi text".
@Composable
fun Base(navController: NavController) {
fun MainBase(navController: NavController) {
Column(
Modifier
@ -103,7 +140,38 @@ fun DropDownSettingsMenu(navController: NavController) {
}
@Composable
fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: Boolean) {
fun RecordingFooterButton(
audioRecorder: AudioRecorder,
navController: NavController,
uiState: ConversationUiState
) {
var isListening: Boolean by remember { mutableStateOf(false) }
var isProcessing: Boolean by remember { mutableStateOf(false) }
// Add a listener for the waveRecorder to record when isListening is true and then process the audio when done listening
audioRecorder.waveRecorder.onStateChangeListener = {
when (it) {
RecorderState.RECORDING -> isListening = true
RecorderState.STOP -> {
thread {
isListening = false
isProcessing = true
processMessage(
processAudio(audioRecorder.getOutputFile()),
navController,
uiState
)
isProcessing = false
audioRecorder.getOutputFile().delete()
}
}
else -> {}
}
}
//We create a row that we align to the bottom center of the parent box
Row(
Modifier
@ -113,8 +181,11 @@ fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: B
) {
//Microphone floating button to manually start/stop listening
FloatingActionButton(onClick = onClick, modifier = Modifier.size(70.dp)) {
if (isRecording) {
FloatingActionButton(
onClick = { if (isListening) audioRecorder.stopRecording() else audioRecorder.startRecording() },
modifier = Modifier.size(70.dp)
) {
if (isListening) {
DotsTyping(7.dp, 3, 300, MaterialTheme.colorScheme.secondary, 2.dp)
} else {
if (isProcessing) {
@ -130,84 +201,27 @@ fun StartRecordingFAB(onClick: () -> Unit, isRecording: Boolean, isProcessing: B
}
}
fun processAudio(audioFile: File): String {
val json = JSONObject(getTextFromAudio(audioFile))
@Composable
fun DisplayMainPage(
navController: NavController,
uiState: ConversationUiState,
audioRecorder: AudioRecorder
) {
//We create a main box with basic padding to avoid having stuff too close to every side.
DefaultBox {
// This column regroup the base and all the conversations (everything except the footer)
Column(Modifier.padding(bottom = 80.dp)) {
Base(navController)
Messages(
messages = uiState.messages,
modifier = Modifier.weight(1f)
)
}
// Finally we add the footer to the bottom center of the main box
Column(
Modifier
.align(Alignment.BottomCenter)
.padding(bottom = 40.dp)
) {
var listening: Boolean by remember { mutableStateOf(false) }
var processing: Boolean by remember { mutableStateOf(false) }
SocketHandler.getSocket().on("message_from_jarvis") { args ->
if (args[0] != null) {
uiState.addMessage(Message(true, args.toString()))
}
}
audioRecorder.waveRecorder.onStateChangeListener = {
when (it) {
RecorderState.RECORDING -> listening = true
RecorderState.STOP -> {
listening = false
processing = true
SocketHandler.processMessage("test", MainActivity().uniqueID)
thread {
val requestOutput = getTextFromAudio(audioRecorder.getOutputFile())
/*val temp = JSONObject()
temp.put("data", "salut je suis bob")
val requestOutput = temp.toString()*/
processing = false
val json = JSONObject(requestOutput)
val sent = json.getString("data")
uiState.addMessage(Message(false, sent))
// Thread.sleep(1000)
// uiState.addMessage(Message(true, json.getString("answer")))
audioRecorder.getOutputFile().delete()
}
}
else -> {}
}
}
StartRecordingFAB(
onClick = { if (listening) audioRecorder.stopRecording() else audioRecorder.startRecording() },
isRecording = listening,
isProcessing = processing
)
}
}
return json.getString("data")
}
fun processMessage(text: String, navController: NavController, uiState: ConversationUiState) {
navController.context.mainExecutor.execute {
SocketHandler.processMessage(
text,
Secure.getString(
navController.context.contentResolver,
Secure.ANDROID_ID
)
)
}
uiState.addMessage(Message(false, text))
}
@Preview(showBackground = true)
@Composable
fun MainPagePreview() {

60
app/src/main/java/ch/broillet/jarvis/android/pages/PermissionsPage.kt
View File

@ -2,11 +2,12 @@ package ch.broillet.jarvis.android.pages
import android.Manifest
import androidx.compose.foundation.layout.*
import androidx.compose.material3.Icon
import androidx.compose.material3.Text
import androidx.compose.material3.*
import androidx.compose.runtime.Composable
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalInspectionMode
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.tooling.preview.Preview
@ -18,7 +19,10 @@ import ch.broillet.jarvis.android.R
import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme
import ch.broillet.jarvis.android.ui.theme.productSansFont
import ch.broillet.jarvis.android.utils.DefaultBox
import ch.broillet.jarvis.android.utils.requestPermissionButton
import ch.broillet.jarvis.android.utils.openAppSettings
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.google.accompanist.permissions.PermissionRequired
import com.google.accompanist.permissions.rememberPermissionState
@Composable
fun DisplayPermissionsPage(navController: NavController) {
@ -54,19 +58,13 @@ fun PermissionsBase() {
modifier = Modifier.padding(top = 10.dp)
)
// List of required permissions to display
PermissionRow(
R.drawable.ic_baseline_mic_24,
stringResource(id = R.string.permission_microphone),
stringResource(id = R.string.permission_microphone_description),
Manifest.permission.RECORD_AUDIO
)
/*PermissionRow(
R.drawable.ic_baseline_folder_open_24,
stringResource(id = R.string.permission_files),
stringResource(id = R.string.permission_files_description),
Manifest.permission.WRITE_EXTERNAL_STORAGE
)*/
}
}
@ -106,8 +104,48 @@ fun PermissionRow(
}
}
requestPermissionButton(permission = permission)
// Remove preview error (only in IDE, has no impact on app)
if (!LocalInspectionMode.current) {
PermissionRequestButton(permission = permission)
}
}
}
@OptIn(ExperimentalPermissionsApi::class)
@Composable
fun PermissionRequestButton(permission: String) {
val permissionState = rememberPermissionState(permission = permission)
val context = LocalContext.current
PermissionRequired(
permissionState = permissionState,
permissionNotGrantedContent = {
Button(onClick = {
permissionState.launchPermissionRequest()
}) {
Text(text = stringResource(id = R.string.permissions_page_grant_permission))
}
},
permissionNotAvailableContent = {
Button(
onClick = { openAppSettings(context) },
colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colorScheme.error)
) {
Text(text = stringResource(id = R.string.permissions_page_permission_denied))
Icon(
painter = painterResource(id = R.drawable.ic_outline_settings_24),
contentDescription = "app settings icon"
)
}
},
content = {
Button(onClick = {}, enabled = false) {
Text(text = stringResource(id = R.string.permission_granted))
}
}
)
}
@Preview(showBackground = true)

49
app/src/main/java/ch/broillet/jarvis/android/utils/PermissionsUtils.kt
View File

@ -1,49 +0,0 @@
package ch.broillet.jarvis.android.utils
import androidx.compose.material.*
import androidx.compose.material3.*
import androidx.compose.runtime.Composable
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.res.painterResource
import androidx.compose.ui.res.stringResource
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.google.accompanist.permissions.PermissionRequired
import com.google.accompanist.permissions.rememberPermissionState
import ch.broillet.jarvis.android.R
@OptIn(ExperimentalPermissionsApi::class)
@Composable
fun requestPermissionButton(permission: String) {
val permissionState = rememberPermissionState(permission = permission)
val context = LocalContext.current
PermissionRequired(
permissionState = permissionState,
permissionNotGrantedContent = {
Button(onClick = {
permissionState.launchPermissionRequest()
}) {
Text(text = stringResource(id = R.string.permissions_page_grant_permission))
}
},
permissionNotAvailableContent = {
Button(
onClick = { openAppSettings(context) },
colors = ButtonDefaults.buttonColors(contentColor = MaterialTheme.colorScheme.error)
) {
Text(text = stringResource(id = R.string.permissions_page_permission_denied))
Icon(
painter = painterResource(id = R.drawable.ic_outline_settings_24),
contentDescription = "app settings icon"
)
}
},
content = {
Button(onClick = {}, enabled = false) {
Text(text = stringResource(id = R.string.permission_granted))
}
}
)
}

1
app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt
View File

@ -6,7 +6,6 @@ import okhttp3.Request
import okhttp3.RequestBody.Companion.asRequestBody
import java.io.File
import java.io.IOException
import java.net.Socket
fun getTextFromAudio(file: File): String {

11
app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt
View File

@ -1,5 +1,7 @@
package ch.broillet.jarvis.android.utils
import ch.broillet.jarvis.android.chat.ConversationUiState
import ch.broillet.jarvis.android.chat.Message
import io.socket.client.IO
import io.socket.client.Socket
import org.json.JSONObject
@ -46,4 +48,13 @@ object SocketHandler {
body.put("uuid", uuid)
getSocket().emit("join", body.toString())
}
fun messageFromJarvis(data: Array<Any>, uiState: ConversationUiState) {
if (data[0].toString().contains("data")) {
val result: JSONObject = data[0] as JSONObject
uiState.addMessage(Message(true, result.getString("data")))
}
}
}

1
app/src/main/java/ch/broillet/jarvis/android/utils/Utils.kt
View File

@ -26,6 +26,7 @@ fun DefaultBox(
}
// Open the application in the system's settings to (i.e) manually give permission
fun openAppSettings(context: Context) {
context.startActivity(
Intent(