Voice-Controlled Computer Vision Assistant (VisionA)
Download original notebook: VisionA
OBJECT
!pip install gtts
!pip install gTTs
!!pip install face-recognition
!git clone https://github.com/cpluz-shrijayan/darknet
from IPython.display import display, Javascript, Image
from google.colab.output import eval_js
from google.colab.patches import cv2_imshow
from base64 import b64decode, b64encode
import cv2
import numpy as np
import PIL
import io
import html
import time
import matplotlib.pyplot as plt
from gtts import gTTS #Import Google Text to Speech
from IPython.display import Audio #Import Audio method from IPython's Display Class
%matplotlib inline
%cd darknet
!sed -i 's/OPENCV=0/OPENCV=1/' Makefile
!sed -i 's/GPU=0/GPU=1/' Makefile
!sed -i 's/CUDNN=0/CUDNN=1/' Makefile
!sed -i 's/CUDNN_HALF=0/CUDNN_HALF=1/' Makefile
!sed -i 's/LIBSO=0/LIBSO=1/' Makefile
!make!wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq" -O yolov4-csp.weights && rm -rf /tmp/cookies.txt
# import darknet functions to perform object detections
from darknet import *
# load in our YOLOv4 architecture network
network, class_names, class_colors = load_network("cfg/yolov4-csp.cfg", "cfg/coco.data", "yolov4-csp.weights")
width = network_width(network)
height = network_height(network)
# darknet helper function to run detection on image
def darknet_helper(img, width, height):
darknet_image = make_image(width, height, 3)
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_resized = cv2.resize(img_rgb, (width, height),interpolation=cv2.INTER_LINEAR)
# get image ratios to convert bounding boxes to proper size
img_height, img_width, _ = img.shape
width_ratio = img_width/width
height_ratio = img_height/height
# run model on darknet style image to get detections
copy_image_from_bytes(darknet_image, img_resized.tobytes())
detections = detect_image(network, class_names, darknet_image)
free_image(darknet_image)
return detections, width_ratio, height_ratiodef js_to_image(js_reply):
"""
Params:
js_reply: JavaScript object containing image from webcam
Returns:
img: OpenCV BGR image
"""
image_bytes = b64decode(js_reply.split(',')[1])
jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)
img = cv2.imdecode(jpg_as_np, flags=1)
return img
def bbox_to_bytes(bbox_array):
"""
Params:
bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.
Returns:
bytes: Base64 image byte string
"""
bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')
iobuf = io.BytesIO()
bbox_PIL.save(iobuf, format='png')
bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))
return bbox_bytesdef take_photo(filename='photo.jpg', quality=0.8):
js = Javascript('''
async function takePhoto(quality) {
const div = document.createElement('div');
const capture = document.createElement('button');
capture.textContent = 'Capture';
div.appendChild(capture);
const video = document.createElement('video');
video.style.display = 'block';
const stream = await navigator.mediaDevices.getUserMedia({video: true});
document.body.appendChild(div);
div.appendChild(video);
video.srcObject = stream;
await video.play();
google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);
await new Promise((resolve) => capture.onclick = resolve);
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext('2d').drawImage(video, 0, 0);
stream
.getVideoTracks()[0].stop();
div.remove();
return canvas.toDataURL('image/jpeg', quality);
}
''')
display(js)
data = eval_js('takePhoto({})'.format(quality))
img = js_to_image(data)
detections, width_ratio, height_ratio = darknet_helper(img, width, height)
l=[]
d={}
for label, confidence, bbox in detections:
left, top, right, bottom = bbox2points(bbox)
left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)
cv2.rectangle(img, (left, top), (right, bottom), class_colors[label], 2)
cv2.putText(img, "{} [{:.2f}]".format(label, float(confidence)),(left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,class_colors[label], 2)
l.append(label)
for j in l:
d[j]=l.count(j)
m1=list(d.keys())
m2=list(d.values())
zz=["There are"]
for z in range(len(m1)):
zz.append(str(m2[z]))
zz.append(m1[z])
zz.append("before you")
global ccc
if len(zz)==2:
ccc="please try again, i cant detect any objects before you"
else:
ccc=str(zz)
cv2.imwrite(filename, img)
return filenamefrom IPython.display import Image
try:
filename = take_photo()
print('Saved to {}'.format(filename))
# Show the image which was just taken.
display(Image(filename))
except Exception as err:
# Errors will be thrown if the user does not have a webcam or if they do not
# grant the page permission to access it.
print(str(err))
### NPL
tts = gTTS(ccc) #Provide the string to convert to speech
tts.save('1.wav') #save the string converted to speech as a .wav file
sound_file = '1.wav'
Audio(sound_file, autoplay=True)LIVE CAM
# JavaScript to properly create our live video stream using our webcam as input
def video_stream():
js = Javascript('''
var video;
var div = null;
var stream;
var captureCanvas;
var imgElement;
var labelElement;
var pendingResolve = null;
var shutdown = false;
function removeDom() {
stream.getVideoTracks()[0].stop();
video.remove();
div.remove();
video = null;
div = null;
stream = null;
imgElement = null;
captureCanvas = null;
labelElement = null;
}
function onAnimationFrame() {
if (!shutdown) {
window.requestAnimationFrame(onAnimationFrame);
}
if (pendingResolve) {
var result = "";
if (!shutdown) {
captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);
result = captureCanvas.toDataURL('image/jpeg', 0.8)
}
var lp = pendingResolve;
pendingResolve = null;
lp(result);
}
}
async function createDom() {
if (div !== null) {
return stream;
}
div = document.createElement('div');
div.style.border = '2px solid black';
div.style.padding = '3px';
div.style.width = '100%';
div.style.maxWidth = '600px';
document.body.appendChild(div);
const modelOut = document.createElement('div');
modelOut.innerHTML = "<span>Status:</span>";
labelElement = document.createElement('span');
labelElement.innerText = 'No data';
labelElement.style.fontWeight = 'bold';
modelOut.appendChild(labelElement);
div.appendChild(modelOut);
video = document.createElement('video');
video.style.display = 'block';
video.width = div.clientWidth - 6;
video.setAttribute('playsinline', '');
video.onclick = () => { shutdown = true; };
stream = await navigator.mediaDevices.getUserMedia(
{video: { facingMode: "environment"}});
div.appendChild(video);
imgElement = document.createElement('img');
imgElement.style.position = 'absolute';
imgElement.style.zIndex = 1;
imgElement.onclick = () => { shutdown = true; };
div.appendChild(imgElement);
const instruction = document.createElement('div');
instruction.innerHTML =
'<span style="color: red; font-weight: bold;">' +
'When finished, click here or on the video to stop this demo</span>';
div.appendChild(instruction);
instruction.onclick = () => { shutdown = true; };
video.srcObject = stream;
await video.play();
captureCanvas = document.createElement('canvas');
captureCanvas.width = 640; //video.videoWidth;
captureCanvas.height = 480; //video.videoHeight;
window.requestAnimationFrame(onAnimationFrame);
return stream;
}
async function stream_frame(label, imgData) {
if (shutdown) {
removeDom();
shutdown = false;
return '';
}
var preCreate = Date.now();
stream = await createDom();
var preShow = Date.now();
if (label != "") {
labelElement.innerHTML = label;
}
if (imgData != "") {
var videoRect = video.getClientRects()[0];
imgElement.style.top = videoRect.top + "px";
imgElement.style.left = videoRect.left + "px";
imgElement.style.width = videoRect.width + "px";
imgElement.style.height = videoRect.height + "px";
imgElement.src = imgData;
}
var preCapture = Date.now();
var result = await new Promise(function(resolve, reject) {
pendingResolve = resolve;
});
shutdown = false;
return {'create': preShow - preCreate,
'show': preCapture - preShow,
'capture': Date.now() - preCapture,
'img': result};
}
''')
display(js)
def video_frame(label, bbox):
data = eval_js('stream_frame("{}", "{}")'.format(label, bbox))
return data# start streaming video from webcam
video_stream()
# label for video
label_html = 'Capturing...'
# initialze bounding box to empty
bbox = ''
count = 0
while True:
js_reply = video_frame(label_html, bbox)
if not js_reply:
break
# convert JS response to OpenCV Image
frame = js_to_image(js_reply["img"])
# create transparent overlay for bounding box
bbox_array = np.zeros([480,640,4], dtype=np.uint8)
# call our darknet helper on video frame
detections, width_ratio, height_ratio = darknet_helper(frame, width, height)
count=0
# loop through detections and draw them on transparent overlay image
for label, confidence, bbox in detections:
count+=1
left, top, right, bottom = bbox2points(bbox)
left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)
bbox_array = cv2.rectangle(bbox_array, (left, top), (right, bottom), class_colors[label], 2)
bbox_array = cv2.putText(bbox_array, "{} [{:.2f}]".format(label, float(confidence)),
(left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
class_colors[label], 2)
bbox_array[:,:,3] = (bbox_array.max(axis = 2) > 0 ).astype(int) * 255
# convert overlay of bbox into bytes
bbox_bytes = bbox_to_bytes(bbox_array)
# update bbox so next frame gets new overlay
bbox = bbox_bytesVoice
!pip install pyttsx3
!pip install SpeechRecognition
!pip install gTTS
!pip install transformers command
!pip install pipwin
!pip install conda!apt install libasound2-dev portaudio19-dev libportaudio2 libportaudiocpp0 ffmpeg
!pip install PyAudio###############################################################################
# CHATBOT #
###############################################################################
# Setup
## for speech-to-text
import speech_recognition as sr
## for text-to-speech
from gtts import gTTS
## for language model
import transformers
## for data
import os
import datetime
import numpy as np
#colab
from io import BytesIO
from base64 import b64decode
from google.colab import output
from IPython.display import Javascript
# Build the AI
class ChatBot():
def __init__(self, name):
print("--- starting up", name, "---")
self.name = name
def speech_to_text(self):
recognizer = sr.Recognizer()
RECORD = """
const sleep = time => new Promise(resolve => setTimeout(resolve, time))
const b2text = blob => new Promise(resolve => {
const reader = new FileReader()
reader.onloadend = e => resolve(e.srcElement.result)
reader.readAsDataURL(blob)
})
var record = time => new Promise(async resolve => {
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
recorder = new MediaRecorder(stream)
chunks = []
recorder.ondataavailable = e => chunks.push(e.data)
recorder.start()
await sleep(time)
recorder.onstop = async ()=>{
blob = new Blob(chunks)
text = await b2text(blob)
resolve(text)
}
recorder.stop()
})
"""
def record(sec=3):
print("Speak Now...")
display(Javascript(RECORD))
sec += 1
s = output.eval_js('record(%d)' % (sec*1000))
print("Done Recording !")
b = b64decode(s.split(',')[1])
return b #byte stream
with sr.Microphone() as mic:
print("listening...")
audio = recognizer.listen(mic)
try:
self.text = recognizer.recognize_google(audio)
print("me --> ", self.text)
except:
print("me --> ERROR")
@staticmethod
def text_to_speech(text):
print("ai --> ", text)
speaker = gTTS(text=text, lang="en", slow=False)
speaker.save("res.mp3")
os.system("afplay res.mp3") #mac->afplay | windows->start
os.remove("res.mp3")
def wake_up(self, text):
return True if self.name in text.lower() else False
@staticmethod
def action_time():
return datetime.datetime.now().time().strftime('%H:%M')
# Run the AI
if __name__ == "__main__":
ai = ChatBot(name="maya")
nlp = transformers.pipeline("conversational", model="microsoft/DialoGPT-medium")
os.environ["TOKENIZERS_PARALLELISM"] = "true"
while True:
ai.speech_to_text()
## wake up
if ai.wake_up(ai.text) is True:
res = "Hello I am Maya the AI, what can I do for you?"
## action time
elif "time" in ai.text:
res = ai.action_time()
## respond politely
elif any(i in ai.text for i in ["thank","thanks"]):
res = np.random.choice(["you're welcome!","anytime!","no problem!","cool!","I'm here if you need me!","peace out!"])
## conversation
else:
chat = nlp(transformers.Conversation(ai.text), pad_token_id=50256)
res = str(chat)
res = res[res.find("bot >> ")+6:].strip()
ai.text_to_speech(res)from io import BytesIO
from base64 import b64decode
from google.colab import output
from IPython.display import Javascript
# all imports
from io import BytesIO
from base64 import b64decode
from google.colab import output
from IPython.display import Javascript
RECORD = """
const sleep = time => new Promise(resolve => setTimeout(resolve, time))
const b2text = blob => new Promise(resolve => {
const reader = new FileReader()
reader.onloadend = e => resolve(e.srcElement.result)
reader.readAsDataURL(blob)
})
var record = time => new Promise(async resolve => {
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
recorder = new MediaRecorder(stream)
chunks = []
recorder.ondataavailable = e => chunks.push(e.data)
recorder.start()
await sleep(time)
recorder.onstop = async ()=>{
blob = new Blob(chunks)
text = await b2text(blob)
resolve(text)
}
recorder.stop()
})
"""
def record(sec=3):
print("Speak Now...")
display(Javascript(RECORD))
sec += 1
s = output.eval_js('record(%d)' % (sec*1000))
print("Done Recording !")
b = b64decode(s.split(',')[1])
return b #byte stream
import speech_recognition as sr
r = sr.Recognizer()
with sr.Microphone() as source:
print("Speak Anything :")
audio = r.listen(source)
try:
text = r.recognize_google(audio)
print("You said : {}".format(text))
except:
print("Sorry could not recognize what you said")import pyttsx3 #pip install pyttsx3
import speech_recognition as sr #pip install speechRecognition
import datetime
import wikipedia #pip install wikipedia
import webbrowser
import os
import smtplib
import screen_brightness_control as sbc
import wolframalpha
import json
import requests
import subprocess
import time
engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
# print(voices[1].id)
engine.setProperty('voice', voices[2].id)
def speak(audio):
engine.say(audio)
engine.runAndWait()
def wishMe():
hour = int(datetime.datetime.now().hour)
if hour>=0 and hour<12:
speak("Good Morning!")
elif hour>=12 and hour<18:
speak("Good Afternoon!")
else:
speak("Good Evening!")
speak(" I am your assistant. How may I help you ?? ")
def takeCommand():
#It takes microphone input from the user and returns string output
r = sr.Recognizer()
with sr.Microphone() as source:
print("Listening...")
r.pause_threshold = 0.7
audio = r.listen(source)
try:
print("Recognizing...")
query = r.recognize_google(audio, language='en-in')
print(f"User said: {query}\n")
except Exception as e:
# print(e)
print("Say that again please...")
return "None"
return query
def sendEmail(to, content):
server = smtplib.SMTP('smtp.gmail.com', 587)
server.ehlo()
server.starttls()
server.login('[email protected]', 'ktmrc690')
server.sendmail('[email protected]', to, content)
server.close()
if __name__ == "__main__":
wishMe()
while True:
# if 1:
query = takeCommand().lower()
# Logic for executing tasks based on query
if 'wikipedia' in query:
speak('Searching Wikipedia...')
query = query.replace("wikipedia", "")
results = wikipedia.summary(query, sentences=2)
speak("According to Wikipedia")
print(results)
speak(results)
elif 'open youtube' in query:
webbrowser.open("youtube.com")
elif 'open google' in query:
webbrowser.open("google.com")
elif 'open stack overflow' in query:
webbrowser.open("stackoverflow.com")
elif 'play music' in query:
music_dir = 'D:\Music'
songs = os.listdir(music_dir)
print(songs)
os.startfile(os.path.join(music_dir, songs[0]))
elif 'the time' in query:
strTime = datetime.datetime.now().strftime("%H:%M:%S")
speak(f"Sir, the time is {strTime}")
elif 'shutdown' in query:
speak("Shutting down laptop")
os.system("shutdown /s /t 1")
elif 'email to me' in query:
try:
speak("What should I say?")
content = takeCommand()
to = "[email protected]"
sendEmail(to, content)
speak("Email has been sent!")
except Exception as e:
print(e)
speak("Sorry Sir. I am not able to send this email")
elif "current brightness" in query:
current_brightness = sbc.get_brightness()
print(f"Current brightness of your device is {current_brightness}")
speak(f"Current brightness of your device is {current_brightness}")
elif "adjust brightness" in query:
speak("What value should brightness be adjusted?")
desired_brightness_1 = takeCommand()
print(desired_brightness_1)
desired_brightness_2 = takeCommand()
print(desired_brightness_2)
desired_brightness = desired_brightness_1 + desired_brightness_2
speak(f"Current brightness adjusted to {desired_brightness}")
print(f"Current brightness adjusted to {desired_brightness}")
sbc.set_brightness(desired_brightness,display = None,verbose_error = False,no_return = False)
elif "weather" in query:
api_key="8ef61edcf1c576d65d836254e11ea420"
base_url="https://api.openweathermap.org/data/2.5/weather?"
speak("whats the city name")
city_name = takeCommand()
complete_url = base_url+"appid="+api_key+"&q="+city_name
response = requests.get(complete_url)
x=response.json()
if x["cod"]!="404":
y=x["main"]
current_temperature = y["temp"]
current_humidiy = y["humidity"]
z = x["weather"]
weather_description = z[0]["description"]
speak(" Temperature in kelvin unit is " +
str(current_temperature) +
"\n humidity in percentage is " +
str(current_humidiy) +
"\n description " +
str(weather_description))
print(" Temperature in kelvin unit = " +
str(current_temperature) +
"\n humidity (in percentage) = " +
str(current_humidiy) +
"\n description = " +
str(weather_description))
else:
speak(" City Not Found ")
elif 'search' in query:
query = query.replace("search", "")
webbrowser.open_new_tab(query)
time.sleep(0.8)
elif 'who are you' in query or 'what can you do' in query:
speak('I am G-one version 1 point O your persoanl assistant. I am programmed to minor tasks like'
'opening youtube,google chrome,gmail and stackoverflow ,predict time,take a photo,search wikipedia,predict weather'
'in different cities , get top headline news from times of india and you can ask me computational or geographical questions too!')
elif "who made you" in query or "who created you" in query:
speak("I was built by AI Lab")
print("I was built by AI Lab")
elif 'news' in query:
try:
speak("Which news article would you like to read ??")
print("Which news article would you like to read ??")
article = takeCommand()
if "Times of india" in article:
webbrowser.open_new_tab("https://timesofindia.indiatimes.com/home/headlines")
speak('Here are some headlines from the Times of India,Happy reading')
time.sleep(0.8)
elif "The hindu" in article:
news = webbrowser.open_new_tab("https://www.thehindu.com/")
speak('Here are some headlines from the hindu,Happy reading')
except:
speak("The news article that you asked for is not available")
print("The news article that you asked for is not available")
# elif "list of monitors" in query:
# monitors = sbc.list_monitors()
# speak(f"List of monitors are {monitors}")
# print(monitors)
elif 'stop listening' in query:
speak("Okay sir. Thank you")
exit()#Face Recognition
import face_recognition
picture_of_me = face_recognition.load_image_file("/content/WIN_20220517_15_08_29_Pro.jpg")
my_face_encoding = face_recognition.face_encodings(picture_of_me)[0]
# my_face_encoding now contains a universal 'encoding' of my facial features that can be compared to any other picture of a face!
unknown_picture = face_recognition.load_image_file("/content/WIN_20220517_15_13_25_Pro.jpg")
unknown_face_encoding = face_recognition.face_encodings(unknown_picture)[0]
# Now we can see the two face encodings are of the same person with `compare_faces`!
results = face_recognition.compare_faces([my_face_encoding], unknown_face_encoding)
if results[0] == True:
print("It's a picture of abi!")
else:
print("It's not a picture of me!")