From be4df87c9901d5b475e97f817526e22658705d63 Mon Sep 17 00:00:00 2001 From: mathieu Date: Fri, 2 Dec 2022 15:01:23 +0100 Subject: [PATCH] websockets stuff --- app/build.gradle | 4 ++ .../broillet/jarvis/android/MainActivity.kt | 15 +++++- .../broillet/jarvis/android/pages/MainPage.kt | 31 ++++++++---- .../jarvis/android/utils/RequestsUtils.kt | 8 ++-- .../jarvis/android/utils/SocketHandler.kt | 47 +++++++++++++++++++ 5 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt diff --git a/app/build.gradle b/app/build.gradle index 6e52d18..b5fc77c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -71,4 +71,8 @@ dependencies { implementation 'com.github.squti:Android-Wave-Recorder:1.7.0' implementation("com.squareup.okhttp3:okhttp:4.9.3") implementation 'com.google.code.gson:gson:2.8.9' + + implementation ('io.socket:socket.io-client:2.0.0') { + exclude group: 'org.json', module: 'json' + } } \ No newline at end of file diff --git a/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt b/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt index bbcbc59..b4ac8cc 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/MainActivity.kt @@ -13,11 +13,23 @@ import androidx.compose.ui.tooling.preview.Preview import androidx.core.view.WindowCompat import ch.broillet.jarvis.android.nav.Navigation import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme +import ch.broillet.jarvis.android.utils.SocketHandler +import java.util.* + class MainActivity : ComponentActivity() { + var uniqueID = UUID.randomUUID().toString() + override fun onCreate(savedInstanceState: Bundle?) { + + // The following lines connects the Android app to the server. + SocketHandler.setSocket() + SocketHandler.establishConnection() + SocketHandler.joinRoom(uniqueID) + super.onCreate(savedInstanceState) + setContent { JarvisclientappTheme { @@ -30,8 +42,7 @@ class MainActivity : ComponentActivity() { // A surface container using the 'background' color from the theme Surface( - modifier = Modifier.fillMaxSize(), - color = MaterialTheme.colorScheme.background + modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background ) { DefaultPreview() } diff --git a/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt b/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt index 36cdc60..91649e4 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/pages/MainPage.kt @@ -1,5 +1,6 @@ package ch.broillet.jarvis.android.pages +import android.os.Looper import androidx.compose.foundation.layout.* import androidx.compose.material.icons.Icons import androidx.compose.material.icons.outlined.Edit @@ -17,6 +18,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.navigation.NavController import androidx.navigation.compose.rememberNavController +import ch.broillet.jarvis.android.MainActivity import ch.broillet.jarvis.android.R import ch.broillet.jarvis.android.audio.AudioRecorder import ch.broillet.jarvis.android.chat.ConversationUiState @@ -25,10 +27,7 @@ import ch.broillet.jarvis.android.chat.Messages import ch.broillet.jarvis.android.nav.Screen import ch.broillet.jarvis.android.ui.theme.JarvisclientappTheme import ch.broillet.jarvis.android.ui.theme.productSansFont -import ch.broillet.jarvis.android.utils.DefaultBox -import ch.broillet.jarvis.android.utils.DotsFlashing -import ch.broillet.jarvis.android.utils.DotsTyping -import ch.broillet.jarvis.android.utils.contactServerWithFileAudioRecording +import ch.broillet.jarvis.android.utils.* import com.github.squti.androidwaverecorder.RecorderState import com.github.squti.androidwaverecorder.WaveRecorder import org.json.JSONObject @@ -37,7 +36,7 @@ import kotlin.concurrent.thread //Draws the base of the main activity, that includes the 3-dots menu and the "hi text". @Composable -fun Base(navController: NavController, uiState: ConversationUiState) { +fun Base(navController: NavController) { Column( Modifier @@ -146,7 +145,7 @@ fun DisplayMainPage( // This column regroup the base and all the conversations (everything except the footer) Column(Modifier.padding(bottom = 80.dp)) { - Base(navController, uiState) + Base(navController) Messages( messages = uiState.messages, @@ -163,22 +162,34 @@ fun DisplayMainPage( var listening: Boolean by remember { mutableStateOf(false) } var processing: Boolean by remember { mutableStateOf(false) } + SocketHandler.getSocket().on("message_from_jarvis") { args -> + if (args[0] != null) { + uiState.addMessage(Message(true, args.toString())) + } + } audioRecorder.waveRecorder.onStateChangeListener = { when (it) { RecorderState.RECORDING -> listening = true RecorderState.STOP -> { listening = false processing = true + + SocketHandler.processMessage("test", MainActivity().uniqueID) + thread { - val requestOutput = - contactServerWithFileAudioRecording(audioRecorder.getOutputFile()) + //val requestOutput = getTextFromAudio(audioRecorder.getOutputFile()) + + val temp = JSONObject() + temp.put("data", "salut je suis bob") + val requestOutput = temp.toString() processing = false val json = JSONObject(requestOutput) - val sent = json.getString("transcription") + val sent = json.getString("data") uiState.addMessage(Message(false, sent)) + // Thread.sleep(1000) // uiState.addMessage(Message(true, json.getString("answer"))) audioRecorder.getOutputFile().delete() @@ -191,7 +202,7 @@ fun DisplayMainPage( StartRecordingFAB( onClick = { if (listening) audioRecorder.stopRecording() else audioRecorder.startRecording() }, isRecording = listening, - isProcessing = processing + isProcessing = processing ) } } diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt index 0376894..22a17bc 100644 --- a/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt +++ b/app/src/main/java/ch/broillet/jarvis/android/utils/RequestsUtils.kt @@ -6,12 +6,14 @@ import okhttp3.Request import okhttp3.RequestBody.Companion.asRequestBody import java.io.File import java.io.IOException +import java.net.Socket -fun contactServerWithFileAudioRecording(file: File): String { + +fun getTextFromAudio(file: File): String { val client = OkHttpClient() val request = Request.Builder() - .url("https://jarvis-server.broillet.ch/process_voice") + .url("https://jarvis-server.broillet.ch/get_text_from_audio") .post(file.asRequestBody("audio/x-wav; charset=utf-8".toMediaType())) .build() @@ -45,4 +47,4 @@ fun contactServerWithFileAudioRecording(file: File): String { if (!response.isSuccessful) throw IOException("Unexpected code $response") return response.body!!.string() }*/ -} \ No newline at end of file +} diff --git a/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt b/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt new file mode 100644 index 0000000..94a8a76 --- /dev/null +++ b/app/src/main/java/ch/broillet/jarvis/android/utils/SocketHandler.kt @@ -0,0 +1,47 @@ +package ch.broillet.jarvis.android.utils + +import io.socket.client.IO +import io.socket.client.Socket +import org.json.JSONObject +import java.net.URISyntaxException + +object SocketHandler { + + lateinit var mSocket: Socket + + @Synchronized + fun setSocket() { + try { + mSocket = IO.socket("https://jarvis-server.broillet.ch") + } catch (_: URISyntaxException) { + } + } + + @Synchronized + fun getSocket(): Socket { + return mSocket + } + + @Synchronized + fun establishConnection() { + mSocket.connect() + } + + @Synchronized + fun closeConnection() { + mSocket.disconnect() + } + + @Synchronized + fun processMessage(message: String, uuid: String) { + val body = JSONObject() + body.put("data", message) + body.put("uuid", uuid) + getSocket().emit("process_message", body) + } + + @Synchronized + fun joinRoom(uuid: String) { + getSocket().emit("join", uuid) + } +} \ No newline at end of file