mathieu/jarvis-server-v2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

websockets stuff

Browse Source
This commit is contained in:
Mathieu 2022-12-02 14:45:17 +00:00
parent 0cfcf494e9
commit 8870aad13a
4 changed files with 100 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
jarvis/api.py
View File

@ -1,30 +1,119 @@
import json import json
import tempfile import tempfile
from threading import Lock
import requests import requests
from flask import request, Flask from flask import Flask, render_template, session, request, \
copy_current_request_context
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect
from jarvis.skills import intent_manager
# Set this variable to "threading", "eventlet" or "gevent" to test the
# different async modes, or leave it set to None for the application to choose
# the best option based on installed packages.
async_mode = None
app = Flask(__name__) app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
thread_lock = Lock()
@app.route('/')
def index():
# return render_template('index.html', async_mode=socketio.async_mode)
return "Welcome to Jarvis Server API !"
@socketio.event
def process_message(message):
message = json.loads(message)
text = message['data']
send_jarvis_message_to_room(text, message['uuid'])
emit('my_response', message)
@socketio.event
def my_broadcast_event(message):
session['receive_count'] = session.get('receive_count', 0) + 1
emit('my_response',
{'data': message['data'], 'count': session['receive_count']},
broadcast=True)
@socketio.event
def join(message):
message = json.loads(message)
join_room(message['uuid'])
@socketio.event
def leave(message):
leave_room(message['uuid'])
emit('my_response','In rooms: ' + ', '.join(rooms()))
@socketio.on('close_room')
def on_close_room(message):
emit('my_response', {'data': 'Room ' + message['room'] + ' is closing.',
'count': session['receive_count']},
to=message['room'])
close_room(message['room'])
@socketio.event
def disconnect_request():
@copy_current_request_context
def can_disconnect():
disconnect()
session['receive_count'] = session.get('receive_count', 0) + 1
# for this emit we use a callback function
# when the callback function is invoked we know that the message has been
# received and it is safe to disconnect
emit('my_response', {'data': 'Disconnected!', 'count': session['receive_count']}, callback=can_disconnect)
@socketio.event
def connect():
global thread
emit('my_response', {'data': 'Connected', 'count': 0})
@socketio.on('disconnect')
def test_disconnect():
print('Client disconnected', request.sid)
def send_user_message_to_room(text, room_id):
socketio.emit('message_from_user', {'data': text, "uuid": room_id}, to=room_id)
def send_jarvis_message_to_room(text, room_id):
socketio.emit('message_from_jarvis', {'data': text, "uuid": room_id}, to=room_id)
# .WAV (i.e.) FILE REQUEST # .WAV (i.e.) FILE REQUEST
@app.route("/process_voice", methods=['POST']) @app.route("/get_text_from_audio", methods=['POST'])
def process_audio_request_android(): def get_text_from_audio():
print("[" + request.remote_addr + "] - New STT request") print("[" + request.remote_addr + "] - New STT request")
audio_temp_file = tempfile.NamedTemporaryFile(prefix='jarvis-audio_', suffix='_client') audio_temp_file = tempfile.NamedTemporaryFile(prefix='jarvis-audio_', suffix='_client')
audio_temp_file.write(request.data) audio_temp_file.write(request.data)
print(audio_temp_file.name) print(audio_temp_file.name)
text = text_recognition_whisperasr(audio_temp_file.name) text = whisper_stt(audio_temp_file.name)
# TODO: send to each skill to answer the questions # TODO: send to each skill to answer the questions
return {"transcription": text, "status": 200} return {"data": text, "uuid": "null"}
"""
@app.route("/process_text", methods=['POST']) @app.route("/process_text", methods=['POST'])
def process_text(): def process_text():
print("[" + request.remote_addr + "] - New TXT request") print("[" + request.remote_addr + "] - New TXT request")
@ -33,11 +122,11 @@ def process_text():
answer = intent_manager.recognise(text, request.headers.get('Client-Ip'), request.headers.get('Client-Port')) answer = intent_manager.recognise(text, request.headers.get('Client-Ip'), request.headers.get('Client-Port'))
return {"transcription": text, "answer": answer} return {"transcription": text, "answer": answer}"""
# send request to whisper-asr server (docker) # send request to whisper-asr server (docker)
def text_recognition_whisperasr(audio_file): def whisper_stt(audio_file):
headers = { headers = {
'accept': 'application/json', 'accept': 'application/json',
# 'Content-Type': 'multipart/form-data', # 'Content-Type': 'multipart/form-data',
@ -59,25 +148,5 @@ def text_recognition_whisperasr(audio_file):
return json.loads(response.text)['text'] return json.loads(response.text)['text']
# NOT IMPLEMENTED RIGHT NOW / to use with local whisper cpp (cpu)
"""
def local_recognition(audio_file, time_of_request):
path = os.path.dirname(get_path_file.__file__)
print("Loading model and recognition")
model = path + "/whisper/models/" + "ggml-small.bin"
os.system(path + "/whisper/main -l fr -t 8 -m " + model + " -f " + audio_file + " -otxt") # + "> /dev/null 2>&1")
output = open(audio_file + ".txt").read()
# time_of_resolution = time.perf_counter()
# print(output + f" - {time_of_resolution - time_of_request:0.4f} seconds")
return jsonify(transcription=output, time=5, answer="WIP...")
"""
def start_server(): def start_server():
app.config['JSON_AS_ASCII'] = False socketio.run(app, host='0.0.0.0', port=6000, allow_unsafe_werkzeug=True)
# TODO: add to config
app.run(port=5000, debug=False, host='0.0.0.0', threaded=True)

3
jarvis/start.py
View File

@ -1,7 +1,6 @@
import api import api
import lingua_franca import lingua_franca
from jarvis.skills.entertainement.jokes import JokesSkill
if __name__ == '__main__': if __name__ == '__main__':
@ -9,7 +8,7 @@ if __name__ == '__main__':
lingua_franca.load_language(lang="fr") lingua_franca.load_language(lang="fr")
# Load skills # Load skills
JokesSkill().register() # JokesSkill().register()
# Start the api endpoint # Start the api endpoint
api.start_server() api.start_server()

128
jarvis/utils/templates/index.html
View File

@ -1,128 +0,0 @@
<!DOCTYPE HTML>
<html>
<head>
<title>Flask-SocketIO Test</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js" integrity="sha512-bLT0Qm9VnAYZDflyKcBaQ2gg0hSYNQrJ8RilYldYQ1FxQYoCLtUjuuRuZo+fjqhx/qtq/1itJ0C2ejDxltZVFg==" crossorigin="anonymous"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/3.0.4/socket.io.js" integrity="sha512-aMGMvNYu8Ue4G+fHa359jcPb1u+ytAF+P2SCb+PxrjCdO3n3ZTxJ30zuH39rimUggmTwmh2u7wvQsDTHESnmfQ==" crossorigin="anonymous"></script>
<script type="text/javascript" charset="utf-8">
$(document).ready(function() {
// Connect to the Socket.IO server.
// The connection URL has the following format, relative to the current page:
// http[s]://<domain>:<port>[/<namespace>]
var socket = io();
// Event handler for new connections.
// The callback function is invoked when a connection with the
// server is established.
socket.on('connect', function() {
socket.emit('my_event', {data: 'I\'m connected!'});
});
// Event handler for server sent data.
// The callback function is invoked whenever the server emits data
// to the client. The data is then displayed in the "Received"
// section of the page.
socket.on('my_response', function(msg, cb) {
$('#log').append('<br>' + $('<div/>').text('Received #' + msg.count + ': ' + msg.data).html());
if (cb)
cb();
});
// Interval function that tests message latency by sending a "ping"
// message. The server then responds with a "pong" message and the
// round trip time is measured.
var ping_pong_times = [];
var start_time;
window.setInterval(function() {
start_time = (new Date).getTime();
$('#transport').text(socket.io.engine.transport.name);
socket.emit('my_ping');
}, 1000);
// Handler for the "pong" message. When the pong is received, the
// time from the ping is stored, and the average of the last 30
// samples is average and displayed.
socket.on('my_pong', function() {
var latency = (new Date).getTime() - start_time;
ping_pong_times.push(latency);
ping_pong_times = ping_pong_times.slice(-30); // keep last 30 samples
var sum = 0;
for (var i = 0; i < ping_pong_times.length; i++)
sum += ping_pong_times[i];
$('#ping-pong').text(Math.round(10 * sum / ping_pong_times.length) / 10);
});
// Handlers for the different forms in the page.
// These accept data from the user and send it to the server in a
// variety of ways
$('form#emit').submit(function(event) {
socket.emit('my_event', {data: $('#emit_data').val()});
return false;
});
$('form#broadcast').submit(function(event) {
socket.emit('my_broadcast_event', {data: $('#broadcast_data').val()});
return false;
});
$('form#join').submit(function(event) {
socket.emit('join', {room: $('#join_room').val()});
return false;
});
$('form#leave').submit(function(event) {
socket.emit('leave', {room: $('#leave_room').val()});
return false;
});
$('form#send_room').submit(function(event) {
socket.emit('my_room_event', {room: $('#room_name').val(), data: $('#room_data').val()});
return false;
});
$('form#close').submit(function(event) {
socket.emit('close_room', {room: $('#close_room').val()});
return false;
});
$('form#disconnect').submit(function(event) {
socket.emit('disconnect_request');
return false;
});
});
</script>
</head>
<body>
<h1>Flask-SocketIO Test</h1>
<p>
Async mode is: <b>{{ async_mode }}</b><br>
Current transport is: <b><span id="transport"></span></b><br>
Average ping/pong latency: <b><span id="ping-pong"></span>ms</b>
</p>
<h2>Send:</h2>
<form id="emit" method="POST" action='#'>
<input type="text" name="emit_data" id="emit_data" placeholder="Message">
<input type="submit" value="Echo">
</form>
<form id="broadcast" method="POST" action='#'>
<input type="text" name="broadcast_data" id="broadcast_data" placeholder="Message">
<input type="submit" value="Broadcast">
</form>
<form id="join" method="POST" action='#'>
<input type="text" name="join_room" id="join_room" placeholder="Room Name">
<input type="submit" value="Join Room">
</form>
<form id="leave" method="POST" action='#'>
<input type="text" name="leave_room" id="leave_room" placeholder="Room Name">
<input type="submit" value="Leave Room">
</form>
<form id="send_room" method="POST" action='#'>
<input type="text" name="room_name" id="room_name" placeholder="Room Name">
<input type="text" name="room_data" id="room_data" placeholder="Message">
<input type="submit" value="Send to Room">
</form>
<form id="close" method="POST" action="#">
<input type="text" name="close_room" id="close_room" placeholder="Room Name">
<input type="submit" value="Close Room">
</form>
<form id="disconnect" method="POST" action="#">
<input type="submit" value="Disconnect">
</form>
<h2>Receive:</h2>
<div id="log"></div>
</body>
</html>

153
jarvis/utils/websocketstest.py
View File

@ -1,153 +0,0 @@
import json
import tempfile
from threading import Lock
import requests
from flask import Flask, render_template, session, request, \
copy_current_request_context
from flask_socketio import SocketIO, emit, join_room, leave_room, \
close_room, rooms, disconnect
from jarvis.skills import intent_manager
# Set this variable to "threading", "eventlet" or "gevent" to test the
# different async modes, or leave it set to None for the application to choose
# the best option based on installed packages.
async_mode = None
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode=async_mode)
thread = None
thread_lock = Lock()
@app.route('/')
def index():
# return render_template('index.html', async_mode=socketio.async_mode)
return "Welcome to Jarvis Server API !"
@socketio.event
def process_message(message):
print(message)
text = message['data']
send_jarvis_message_to_room(text, message['uuid'])
emit('my_response', message)
@socketio.event
def my_broadcast_event(message):
session['receive_count'] = session.get('receive_count', 0) + 1
emit('my_response',
{'data': message['data'], 'count': session['receive_count']},
broadcast=True)
@socketio.event
def join(message):
join_room(message['room'])
emit('my_response', join(rooms()))
@socketio.event
def leave(message):
leave_room(message['room'])
emit('my_response','In rooms: ' + ', '.join(rooms()))
@socketio.on('close_room')
def on_close_room(message):
emit('my_response', {'data': 'Room ' + message['room'] + ' is closing.',
'count': session['receive_count']},
to=message['room'])
close_room(message['room'])
@socketio.event
def disconnect_request():
@copy_current_request_context
def can_disconnect():
disconnect()
session['receive_count'] = session.get('receive_count', 0) + 1
# for this emit we use a callback function
# when the callback function is invoked we know that the message has been
# received and it is safe to disconnect
emit('my_response', {'data': 'Disconnected!', 'count': session['receive_count']}, callback=can_disconnect)
@socketio.event
def connect():
global thread
emit('my_response', {'data': 'Connected', 'count': 0})
@socketio.on('disconnect')
def test_disconnect():
print('Client disconnected', request.sid)
def send_user_message_to_room(text, room_id):
socketio.emit('message_from_user', {'data': text, "uuid": room_id}, to=room_id)
def send_jarvis_message_to_room(text, room_id):
socketio.emit('message_from_jarvis', {'data': text, "uuid": room_id}, to=room_id)
# .WAV (i.e.) FILE REQUEST
@app.route("/get_text_from_audio", methods=['POST'])
def get_text_from_audio():
print("[" + request.remote_addr + "] - New STT request")
audio_temp_file = tempfile.NamedTemporaryFile(prefix='jarvis-audio_', suffix='_client')
audio_temp_file.write(request.data)
print(audio_temp_file.name)
text = whisper_stt(audio_temp_file.name)
# TODO: send to each skill to answer the questions
return {"data": text, "uuid": "null"}
"""
@app.route("/process_text", methods=['POST'])
def process_text():
print("[" + request.remote_addr + "] - New TXT request")
text = request.values['text']
answer = intent_manager.recognise(text, request.headers.get('Client-Ip'), request.headers.get('Client-Port'))
return {"transcription": text, "answer": answer}"""
# send request to whisper-asr server (docker)
def whisper_stt(audio_file):
headers = {
'accept': 'application/json',
# 'Content-Type': 'multipart/form-data',
}
params = {
'task': 'transcribe',
# TODO: add to config
'language': 'fr',
'output': 'json',
}
files = {
'audio_file': open(audio_file, 'rb'),
}
# TODO: add to config
response = requests.post('https://whisper.broillet.ch/asr', params=params, headers=headers, files=files)
return json.loads(response.text)['text']
if __name__ == '__main__':
socketio.run(app, host='0.0.0.0', port=5000, allow_unsafe_werkzeug=True)