From f8796c42065b70f4c7b0fceacfe63d9835d4f024 Mon Sep 17 00:00:00 2001 From: Mathieu B Date: Mon, 28 Nov 2022 21:35:38 +0100 Subject: [PATCH] Basic for whisper support (via jarvis-server and whisper-asr) --- .idea/misc.xml | 1 + .../jarvis/android/pages/MainPage.kt | 19 ++++++----- .../jarvis/android/utils/RequestsUtils.kt | 32 +++++++++++++++++-- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 7b44285..36e412f 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -45,6 +45,7 @@ + diff --git a/app/src/main/java/ch/mathieubroillet/jarvis/android/pages/MainPage.kt b/app/src/main/java/ch/mathieubroillet/jarvis/android/pages/MainPage.kt index dcdaf70..26c5c04 100644 --- a/app/src/main/java/ch/mathieubroillet/jarvis/android/pages/MainPage.kt +++ b/app/src/main/java/ch/mathieubroillet/jarvis/android/pages/MainPage.kt @@ -7,21 +7,25 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource +import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.DpOffset import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.navigation.NavController +import androidx.navigation.compose.rememberNavController import ch.mathieubroillet.jarvis.android.R import ch.mathieubroillet.jarvis.android.audio.AudioRecorder import ch.mathieubroillet.jarvis.android.chat.ConversationUiState import ch.mathieubroillet.jarvis.android.chat.Message import ch.mathieubroillet.jarvis.android.chat.Messages import ch.mathieubroillet.jarvis.android.nav.Screen +import ch.mathieubroillet.jarvis.android.ui.theme.JarvisComposeTheme import ch.mathieubroillet.jarvis.android.ui.theme.productSansFont import ch.mathieubroillet.jarvis.android.utils.DefaultBox import ch.mathieubroillet.jarvis.android.utils.IconAlertDialogTextField import ch.mathieubroillet.jarvis.android.utils.contactServerWithFileAudioRecording import com.github.squti.androidwaverecorder.RecorderState +import com.github.squti.androidwaverecorder.WaveRecorder import org.json.JSONObject import kotlin.concurrent.thread @@ -150,13 +154,12 @@ fun DisplayMainPage( val requestOutput = contactServerWithFileAudioRecording(audioRecorder.getOutputFile()) - val json: JSONObject = JSONObject(requestOutput) - val sent = JSONObject(requestOutput).getString("sent").replace("\"", "") - .replace("[", "").replace("]", "").replace(",", " ") - sent.replaceFirstChar { sent.first().uppercase() } + val json = JSONObject(requestOutput) + val sent = json.getString("transcription") + uiState.addMessage(Message(false, sent)) Thread.sleep(1000) - uiState.addMessage(Message(true, json.getString("response"))) + uiState.addMessage(Message(true, json.getString("answer"))) audioRecorder.getOutputFile().delete() } } @@ -172,14 +175,14 @@ fun DisplayMainPage( } } -/*@Preview(showBackground = true) +@Preview(showBackground = true) @Composable fun MainPagePreview() { JarvisComposeTheme { DisplayMainPage( rememberNavController(), ConversationUiState( listOf(Message(true, stringResource(id = R.string.demo_message_1))) - ), null + ), audioRecorder = AudioRecorder("", WaveRecorder("")) ) } -}*/ \ No newline at end of file +} \ No newline at end of file diff --git a/app/src/main/java/ch/mathieubroillet/jarvis/android/utils/RequestsUtils.kt b/app/src/main/java/ch/mathieubroillet/jarvis/android/utils/RequestsUtils.kt index d059402..13081ed 100644 --- a/app/src/main/java/ch/mathieubroillet/jarvis/android/utils/RequestsUtils.kt +++ b/app/src/main/java/ch/mathieubroillet/jarvis/android/utils/RequestsUtils.kt @@ -1,9 +1,11 @@ package ch.mathieubroillet.jarvis.android.utils +import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.asRequestBody +import okhttp3.internal.http1.HeadersReader import java.io.File import java.io.IOException @@ -11,12 +13,38 @@ fun contactServerWithFileAudioRecording(file: File): String { val client = OkHttpClient() val request = Request.Builder() - .url("http://192.168.1.130:5000/process_audio_request_file") - .post(file.asRequestBody("audio/mpeg; charset=utf-8".toMediaType())) + .url("https://whisper.broillet.ch/process_audio_request_file") + .post(file.asRequestBody("audio/x-wav; charset=utf-8".toMediaType())) .build() + client.newCall(request).execute().use { response -> if (!response.isSuccessful) throw IOException("Unexpected code $response") return response.body!!.string() } + + /* # SEND STT REQUEST DIRECTLY TO WHISPER ASR WITHOUT JARVIS SERVER API + val okHttpClient = OkHttpClient() + .newBuilder() + .connectTimeout(60, TimeUnit.SECONDS) + .build() + + val fileBody: RequestBody = RequestBody.create("audio/x-wav".toMediaTypeOrNull(), file.readBytes()) + + val multipartBody: MultipartBody = MultipartBody.Builder() + .setType(MultipartBody.FORM) // Header to show we are sending a Multipart Form Data + .addFormDataPart("audio_file", file.name, fileBody) // file param + .build() + + val request = Request.Builder() + .addHeader("Accept", "application/json") + .addHeader("Content-Type", "multipart/form-data") + .url("http://192.168.1.208:9000/asr?task=transcribe&language=fr&output=json") + .post(multipartBody) + .build() + + okHttpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) throw IOException("Unexpected code $response") + return response.body!!.string() + }*/ } \ No newline at end of file