(IMPORTANT) Removed old AI training model and old intents
This commit is contained in:
Mathieu B
parent
06f8d60d26
commit
6e28b7076e
@ -1,19 +0,0 @@
|
|||||||
import torch.nn as nn
|
|
||||||
|
|
||||||
|
|
||||||
class NeuralNet(nn.Module):
|
|
||||||
def __init__(self, input_size, hidden_size, num_classes):
|
|
||||||
super(NeuralNet, self).__init__()
|
|
||||||
self.l1 = nn.Linear(input_size, hidden_size)
|
|
||||||
self.l2 = nn.Linear(hidden_size, hidden_size)
|
|
||||||
self.l3 = nn.Linear(hidden_size, num_classes)
|
|
||||||
self.relu = nn.ReLU()
|
|
||||||
|
|
||||||
def forward(self, x):
|
|
||||||
out = self.l1(x)
|
|
||||||
out = self.relu(out)
|
|
||||||
out = self.l2(out)
|
|
||||||
out = self.relu(out)
|
|
||||||
out = self.l3(out)
|
|
||||||
# no activation and no softmax at the end
|
|
||||||
return out
|
|
||||||
@ -1,52 +0,0 @@
|
|||||||
import nltk
|
|
||||||
import numpy as np
|
|
||||||
from nltk.stem.porter import PorterStemmer
|
|
||||||
|
|
||||||
from jarvis.utils import languages_utils
|
|
||||||
|
|
||||||
stemmer = PorterStemmer()
|
|
||||||
|
|
||||||
|
|
||||||
# TODO : have a look to replace nltk by spacy or the other way (use only one of them)
|
|
||||||
|
|
||||||
def tokenize(sentence):
|
|
||||||
"""
|
|
||||||
split sentence into array of words/tokens
|
|
||||||
a token can be a word or punctuation character, or number
|
|
||||||
"""
|
|
||||||
# English, Danish, Estonian, French, Greek, Norwegian, Portuguese, Spanish, Turkish,
|
|
||||||
# Czech, Dutch, Finnish, German, Italian, Polish, Slovene, and Swedish
|
|
||||||
|
|
||||||
return nltk.word_tokenize(sentence,
|
|
||||||
language=languages_utils.get_language_full_name())
|
|
||||||
|
|
||||||
|
|
||||||
def stem(word):
|
|
||||||
"""
|
|
||||||
stemming = find the root form of the word
|
|
||||||
examples:
|
|
||||||
words = ["organize", "organizes", "organizing"]
|
|
||||||
words = [stem(w) for w in words]
|
|
||||||
-> ["organ", "organ", "organ"]
|
|
||||||
"""
|
|
||||||
return stemmer.stem(word.lower())
|
|
||||||
|
|
||||||
|
|
||||||
def bag_of_words(tokenized_sentence, words):
|
|
||||||
"""
|
|
||||||
return bag of words array:
|
|
||||||
1 for each known word that exists in the sentence, 0 otherwise
|
|
||||||
example:
|
|
||||||
sentence = ["hello", "how", "are", "you"]
|
|
||||||
words = ["hi", "hello", "I", "you", "bye", "thank", "cool"]
|
|
||||||
bog = [ 0 , 1 , 0 , 1 , 0 , 0 , 0]
|
|
||||||
"""
|
|
||||||
# stem each word
|
|
||||||
sentence_words = [stem(word) for word in tokenized_sentence]
|
|
||||||
# initialize bag with 0 for each word
|
|
||||||
bag = np.zeros(len(words), dtype='float32')
|
|
||||||
for idx, w in enumerate(words):
|
|
||||||
if w in sentence_words:
|
|
||||||
bag[idx] = 1
|
|
||||||
|
|
||||||
return bag
|
|
||||||
@ -1,60 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
import torch
|
|
||||||
from unidecode import unidecode
|
|
||||||
|
|
||||||
from jarvis import get_path_file
|
|
||||||
from jarvis.ia.model import NeuralNet
|
|
||||||
from jarvis.ia.nltk_utils import bag_of_words, tokenize
|
|
||||||
|
|
||||||
print("Loading, might take a few seconds...")
|
|
||||||
|
|
||||||
path = os.path.dirname(get_path_file.__file__)
|
|
||||||
|
|
||||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
||||||
|
|
||||||
file = path + "/ia/trained_model.pth"
|
|
||||||
data = torch.load(file, map_location=device)
|
|
||||||
|
|
||||||
input_size = data["input_size"]
|
|
||||||
hidden_size = data["hidden_size"]
|
|
||||||
output_size = data["output_size"]
|
|
||||||
all_words = data['all_words']
|
|
||||||
tags = data['tags']
|
|
||||||
model_state = data["model_state"]
|
|
||||||
|
|
||||||
model = NeuralNet(input_size, hidden_size, output_size).to(device)
|
|
||||||
model.load_state_dict(model_state)
|
|
||||||
model.eval()
|
|
||||||
|
|
||||||
|
|
||||||
def get_tag_for_sentence(input_sentence):
|
|
||||||
"""
|
|
||||||
Return the matching tag of the input_sentence given in parameter.
|
|
||||||
It usually is what the STT engine recognise or what the user's type when using no-voice mode
|
|
||||||
|
|
||||||
Parameters
|
|
||||||
----------
|
|
||||||
input_sentence is your sentence
|
|
||||||
|
|
||||||
Returns tag from the skills.json file
|
|
||||||
-------
|
|
||||||
|
|
||||||
"""
|
|
||||||
sentence = unidecode(input_sentence) # convert accent to better recognition
|
|
||||||
sentence = tokenize(sentence)
|
|
||||||
X = bag_of_words(sentence, all_words)
|
|
||||||
X = X.reshape(1, X.shape[0])
|
|
||||||
X = torch.from_numpy(X).to(device)
|
|
||||||
|
|
||||||
output = model(X)
|
|
||||||
_, predicted = torch.max(output, dim=1)
|
|
||||||
|
|
||||||
tag = tags[predicted.item()]
|
|
||||||
|
|
||||||
probs = torch.softmax(output, dim=1)
|
|
||||||
prob = probs[0][predicted.item()]
|
|
||||||
if prob.item() > 0.75 and len(sentence) > 2:
|
|
||||||
return tag
|
|
||||||
else:
|
|
||||||
return 'dont_understand'
|
|
||||||
@ -1,138 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
import torch.nn as nn
|
|
||||||
from torch.utils.data import Dataset, DataLoader
|
|
||||||
|
|
||||||
from jarvis import get_path_file
|
|
||||||
from jarvis.ia.model import NeuralNet
|
|
||||||
from jarvis.ia.nltk_utils import bag_of_words, tokenize, stem
|
|
||||||
from jarvis.utils import intents_utils
|
|
||||||
|
|
||||||
path = os.path.dirname(get_path_file.__file__)
|
|
||||||
|
|
||||||
|
|
||||||
def train():
|
|
||||||
intents_utils.register_all_intents() # important
|
|
||||||
all_intents_patterns = intents_utils.get_all_patterns()
|
|
||||||
|
|
||||||
all_words = []
|
|
||||||
tags = []
|
|
||||||
xy = []
|
|
||||||
# loop through each sentence in our skills patterns
|
|
||||||
for intent in all_intents_patterns:
|
|
||||||
tag = intent
|
|
||||||
# add to tag list
|
|
||||||
tags.append(tag)
|
|
||||||
|
|
||||||
for pattern in all_intents_patterns[intent]:
|
|
||||||
# tokenize each word in the sentence
|
|
||||||
w = tokenize(pattern)
|
|
||||||
# add to our words list
|
|
||||||
all_words.extend(w)
|
|
||||||
# add to xy pair
|
|
||||||
xy.append((w, tag))
|
|
||||||
|
|
||||||
# stem and lower each word
|
|
||||||
ignore_words = ['?', '.', '!']
|
|
||||||
all_words = [stem(w) for w in all_words if w not in ignore_words]
|
|
||||||
# remove duplicates and sort
|
|
||||||
all_words = sorted(set(all_words))
|
|
||||||
tags = sorted(set(tags))
|
|
||||||
|
|
||||||
print(len(xy), "patterns")
|
|
||||||
print(len(tags), "tags:", tags)
|
|
||||||
print(len(all_words), "unique stemmed words:", all_words)
|
|
||||||
|
|
||||||
# create training data
|
|
||||||
X_train = []
|
|
||||||
y_train = []
|
|
||||||
for (pattern_sentence, tag) in xy:
|
|
||||||
# X: bag of words for each pattern_sentence
|
|
||||||
bag = bag_of_words(pattern_sentence, all_words)
|
|
||||||
X_train.append(bag)
|
|
||||||
# y: PyTorch CrossEntropyLoss needs only class labels, not one-hot
|
|
||||||
label = tags.index(tag)
|
|
||||||
y_train.append(label)
|
|
||||||
|
|
||||||
X_train = np.array(X_train)
|
|
||||||
y_train = np.array(y_train)
|
|
||||||
|
|
||||||
# Hyper-parameters
|
|
||||||
num_epochs = 1000
|
|
||||||
batch_size = 8
|
|
||||||
learning_rate = 0.001
|
|
||||||
input_size = len(X_train[0])
|
|
||||||
hidden_size = 8
|
|
||||||
output_size = len(tags)
|
|
||||||
print(input_size, output_size)
|
|
||||||
|
|
||||||
class ChatDataset(Dataset):
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.n_samples = len(X_train)
|
|
||||||
self.x_data = X_train
|
|
||||||
self.y_data = y_train
|
|
||||||
|
|
||||||
# support indexing such that dataset[i] can be used to get i-th sample
|
|
||||||
def __getitem__(self, index):
|
|
||||||
return self.x_data[index], self.y_data[index]
|
|
||||||
|
|
||||||
# we can call len(dataset) to return the size
|
|
||||||
def __len__(self):
|
|
||||||
return self.n_samples
|
|
||||||
|
|
||||||
dataset = ChatDataset()
|
|
||||||
train_loader = DataLoader(dataset=dataset,
|
|
||||||
batch_size=batch_size,
|
|
||||||
shuffle=True,
|
|
||||||
num_workers=0)
|
|
||||||
|
|
||||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
||||||
|
|
||||||
model = NeuralNet(input_size, hidden_size, output_size).to(device)
|
|
||||||
|
|
||||||
# Loss and optimizer
|
|
||||||
criterion = nn.CrossEntropyLoss()
|
|
||||||
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
|
|
||||||
|
|
||||||
# Train the model
|
|
||||||
for epoch in range(num_epochs):
|
|
||||||
for (words, labels) in train_loader:
|
|
||||||
words = words.to(device)
|
|
||||||
labels = labels.to(dtype=torch.long).to(device)
|
|
||||||
|
|
||||||
# Forward pass
|
|
||||||
outputs = model(words)
|
|
||||||
# if y would be one-hot, we must apply
|
|
||||||
# labels = torch.max(labels, 1)[1]
|
|
||||||
loss = criterion(outputs, labels)
|
|
||||||
|
|
||||||
# Backward and optimize
|
|
||||||
optimizer.zero_grad()
|
|
||||||
loss.backward()
|
|
||||||
optimizer.step()
|
|
||||||
|
|
||||||
if (epoch + 1) % 100 == 0:
|
|
||||||
print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {loss.item():.4f}')
|
|
||||||
|
|
||||||
print(f'Final loss: {loss.item():.4f}')
|
|
||||||
|
|
||||||
data = {
|
|
||||||
"model_state": model.state_dict(),
|
|
||||||
"input_size": input_size,
|
|
||||||
"hidden_size": hidden_size,
|
|
||||||
"output_size": output_size,
|
|
||||||
"all_words": all_words,
|
|
||||||
"tags": tags
|
|
||||||
}
|
|
||||||
|
|
||||||
file = path + "/ia/trained_model.pth"
|
|
||||||
torch.save(data, file)
|
|
||||||
|
|
||||||
print(f'Training complete. file saved to {file}')
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
train()
|
|
||||||
Binary file not shown.
@ -1,98 +0,0 @@
|
|||||||
import glob
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
|
|
||||||
from jarvis import get_path_file
|
|
||||||
from jarvis.utils import languages_utils
|
|
||||||
|
|
||||||
all_intents = dict()
|
|
||||||
path = os.path.dirname(get_path_file.__file__)
|
|
||||||
|
|
||||||
|
|
||||||
def register_all_intents():
|
|
||||||
global all_intents
|
|
||||||
|
|
||||||
result = {}
|
|
||||||
|
|
||||||
files = glob.glob(path + "/skills/**/info.json", recursive=True)
|
|
||||||
for f in files:
|
|
||||||
with open(f, "rb") as infile:
|
|
||||||
intent_info_json = json.load(infile)
|
|
||||||
intents_in_info = intent_info_json['intents']
|
|
||||||
intent_path = str(f).replace('info.json', '')
|
|
||||||
|
|
||||||
for intent in intents_in_info:
|
|
||||||
result[intent] = intent_path
|
|
||||||
|
|
||||||
all_intents = result
|
|
||||||
|
|
||||||
|
|
||||||
def get_all_intents():
|
|
||||||
if len(all_intents) >= 1:
|
|
||||||
return all_intents
|
|
||||||
else:
|
|
||||||
register_all_intents()
|
|
||||||
return get_all_intents()
|
|
||||||
|
|
||||||
|
|
||||||
def get_all_patterns():
|
|
||||||
all_patterns = {}
|
|
||||||
|
|
||||||
# need to run register first
|
|
||||||
if not all_intents:
|
|
||||||
print("Warning : No intent found at all, don't forget to register them!")
|
|
||||||
return {}
|
|
||||||
|
|
||||||
for intent in all_intents:
|
|
||||||
all_patterns[intent] = get_patterns(intent)
|
|
||||||
|
|
||||||
return all_patterns
|
|
||||||
|
|
||||||
|
|
||||||
def get_patterns(intent_tag):
|
|
||||||
if exists(intent_tag):
|
|
||||||
patterns = get_lang_for_intent(intent_tag).get(intent_tag).get('patterns')
|
|
||||||
return patterns
|
|
||||||
else:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def get_path(intent_tag):
|
|
||||||
if exists(intent_tag):
|
|
||||||
return get_all_intents().get(intent_tag)
|
|
||||||
|
|
||||||
|
|
||||||
def get_response(intent_tag):
|
|
||||||
if exists(intent_tag):
|
|
||||||
responses = get_responses(intent_tag)
|
|
||||||
return random.choice(responses)
|
|
||||||
|
|
||||||
|
|
||||||
def get_responses(intent_tag):
|
|
||||||
if exists(intent_tag):
|
|
||||||
responses = get_lang_for_intent(intent_tag).get(intent_tag).get('responses')
|
|
||||||
return responses
|
|
||||||
else:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def get_lang_for_intent(intent_tag):
|
|
||||||
# first we check the intent
|
|
||||||
if exists(intent_tag):
|
|
||||||
lang_path = str(get_all_intents().get(intent_tag))
|
|
||||||
lang_path = lang_path + 'lang/' + languages_utils.get_language() + '.json'
|
|
||||||
|
|
||||||
if os.path.exists(lang_path):
|
|
||||||
lang_file = open(lang_path)
|
|
||||||
json_lang = json.load(lang_file)
|
|
||||||
return json_lang
|
|
||||||
else:
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def exists(intent_tag):
|
|
||||||
if intent_tag in get_all_intents():
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
Reference in New Issue
Block a user