{"cells":[{"cell_type":"markdown","metadata":{"id":"m2qjB8dJnTCb"},"source":["# OBJECT"]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4Q-5i5VTnWHi","executionInfo":{"status":"ok","timestamp":1664183274635,"user_tz":-330,"elapsed":37800,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}},"outputId":"07007d5a-e3c9-4eb3-e202-4c2e08387c81"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting gtts\n","  Downloading gTTS-2.2.4-py3-none-any.whl (26 kB)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gtts) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from gtts) (7.1.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from gtts) (2.23.0)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->gtts) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->gtts) (2022.6.15)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->gtts) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->gtts) (2.10)\n","Installing collected packages: gtts\n","Successfully installed gtts-2.2.4\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: gTTs in /usr/local/lib/python3.7/dist-packages (2.2.4)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from gTTs) (7.1.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from gTTs) (2.23.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gTTs) (1.15.0)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->gTTs) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->gTTs) (2.10)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->gTTs) (3.0.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->gTTs) (2022.6.15)\n","Cloning into 'darknet'...\n","fatal: could not read Username for 'https://github.com': No such device or address\n","[Errno 2] No such file or directory: 'darknet'\n","/content\n","sed: can't read Makefile: No such file or directory\n","sed: can't read Makefile: No such file or directory\n","sed: can't read Makefile: No such file or directory\n","sed: can't read Makefile: No such file or directory\n","sed: can't read Makefile: No such file or directory\n","make: *** No targets specified and no makefile found.  Stop.\n"]}],"source":["!pip install gtts\n","!pip install gTTs\n","!!pip install face-recognition\n","!git clone https://github.com/cpluz-shrijayan/darknet\n","from IPython.display import display, Javascript, Image\n","from google.colab.output import eval_js\n","from google.colab.patches import cv2_imshow\n","from base64 import b64decode, b64encode\n","import cv2\n","import numpy as np\n","import PIL\n","import io\n","import html\n","import time\n","import matplotlib.pyplot as plt\n","from gtts import gTTS #Import Google Text to Speech\n","from IPython.display import Audio #Import Audio method from IPython's Display Class\n","%matplotlib inline\n","\n","%cd darknet\n","!sed -i 's/OPENCV=0/OPENCV=1/' Makefile\n","!sed -i 's/GPU=0/GPU=1/' Makefile\n","!sed -i 's/CUDNN=0/CUDNN=1/' Makefile\n","!sed -i 's/CUDNN_HALF=0/CUDNN_HALF=1/' Makefile\n","!sed -i 's/LIBSO=0/LIBSO=1/' Makefile\n","!make"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"h9oyiETcnrci","outputId":"d19332f7-6b2f-4df6-9283-0db7727338e5","executionInfo":{"status":"ok","timestamp":1664183317694,"user_tz":-330,"elapsed":4763,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["--2022-09-26 09:08:32--  https://docs.google.com/uc?export=download&confirm=t&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq\n","Resolving docs.google.com (docs.google.com)... 64.233.191.113, 64.233.191.102, 64.233.191.100, ...\n","Connecting to docs.google.com (docs.google.com)|64.233.191.113|:443... connected.\n","HTTP request sent, awaiting response... 303 See Other\n","Location: https://doc-14-84-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/sik92rbbcc62egd5maeqt1p5id45q17b/1664183250000/17800843676226924807/*/1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq?e=download&uuid=e0685144-c99d-40a5-ba03-bb856631cf95 [following]\n","Warning: wildcards not supported in HTTP.\n","--2022-09-26 09:08:33--  https://doc-14-84-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/sik92rbbcc62egd5maeqt1p5id45q17b/1664183250000/17800843676226924807/*/1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq?e=download&uuid=e0685144-c99d-40a5-ba03-bb856631cf95\n","Resolving doc-14-84-docs.googleusercontent.com (doc-14-84-docs.googleusercontent.com)... 108.177.120.132, 2607:f8b0:4001:c18::84\n","Connecting to doc-14-84-docs.googleusercontent.com (doc-14-84-docs.googleusercontent.com)|108.177.120.132|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 211944840 (202M) [application/octet-stream]\n","Saving to: ‘yolov4-csp.weights’\n","\n","yolov4-csp.weights  100%[===================>] 202.13M  53.9MB/s    in 3.7s    \n","\n","2022-09-26 09:08:37 (53.9 MB/s) - ‘yolov4-csp.weights’ saved [211944840/211944840]\n","\n"]}],"source":["!wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq\" -O yolov4-csp.weights && rm -rf /tmp/cookies.txt"]},{"cell_type":"code","execution_count":3,"metadata":{"id":"p7Wbva7Ont7k","colab":{"base_uri":"https://localhost:8080/","height":391},"executionInfo":{"status":"error","timestamp":1664183320569,"user_tz":-330,"elapsed":417,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}},"outputId":"ca9819ce-8e60-4000-b89f-f7a1bf2387e6"},"outputs":[{"output_type":"error","ename":"ModuleNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m<ipython-input-3-4a811324198f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# import darknet functions to perform object detections\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdarknet\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# load in our YOLOv4 architecture network\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mnetwork\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_colors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cfg/yolov4-csp.cfg\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"cfg/coco.data\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"yolov4-csp.weights\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mwidth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnetwork_width\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'darknet'","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}],"source":["\n","# import darknet functions to perform object detections\n","from darknet import *\n","# load in our YOLOv4 architecture network\n","network, class_names, class_colors = load_network(\"cfg/yolov4-csp.cfg\", \"cfg/coco.data\", \"yolov4-csp.weights\")\n","width = network_width(network)\n","height = network_height(network)\n","\n","# darknet helper function to run detection on image\n","def darknet_helper(img, width, height):\n","  darknet_image = make_image(width, height, 3)\n","  img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n","  img_resized = cv2.resize(img_rgb, (width, height),interpolation=cv2.INTER_LINEAR)\n","\n","  # get image ratios to convert bounding boxes to proper size\n","  img_height, img_width, _ = img.shape\n","  width_ratio = img_width/width\n","  height_ratio = img_height/height\n","\n","  # run model on darknet style image to get detections\n","  copy_image_from_bytes(darknet_image, img_resized.tobytes())\n","  detections = detect_image(network, class_names, darknet_image)\n","  free_image(darknet_image)\n","  return detections, width_ratio, height_ratio"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"R3Xsr7Odnt0v"},"outputs":[],"source":["def js_to_image(js_reply):\n","  \"\"\"\n","  Params:\n","          js_reply: JavaScript object containing image from webcam\n","  Returns:\n","          img: OpenCV BGR image\n","  \"\"\"\n","  image_bytes = b64decode(js_reply.split(',')[1])\n","  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)\n","  img = cv2.imdecode(jpg_as_np, flags=1)\n","  return img\n","def bbox_to_bytes(bbox_array):\n","  \"\"\"\n","  Params:\n","          bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.\n","  Returns:\n","        bytes: Base64 image byte string\n","  \"\"\"\n","  bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')\n","  iobuf = io.BytesIO()\n","  bbox_PIL.save(iobuf, format='png')\n","  bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))\n","  return bbox_bytes"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Ivfdsij2ntpC"},"outputs":[],"source":["def take_photo(filename='photo.jpg', quality=0.8):\n","  js = Javascript('''\n","    async function takePhoto(quality) {\n","      const div = document.createElement('div');\n","      const capture = document.createElement('button');\n","      capture.textContent = 'Capture';\n","      div.appendChild(capture);\n","      const video = document.createElement('video');\n","      video.style.display = 'block';\n","      const stream = await navigator.mediaDevices.getUserMedia({video: true});\n","      document.body.appendChild(div);\n","      div.appendChild(video);\n","      video.srcObject = stream;\n","      await video.play();\n","      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n","      await new Promise((resolve) => capture.onclick = resolve);\n","      const canvas = document.createElement('canvas');\n","      canvas.width = video.videoWidth;\n","      canvas.height = video.videoHeight;\n","      canvas.getContext('2d').drawImage(video, 0, 0);\n","      stream\n","      .getVideoTracks()[0].stop();\n","      div.remove();\n","      return canvas.toDataURL('image/jpeg', quality);\n","    }\n","    ''')\n","  display(js)\n","  data = eval_js('takePhoto({})'.format(quality))\n","  img = js_to_image(data) \n","  detections, width_ratio, height_ratio = darknet_helper(img, width, height)\n","  l=[]\n","  d={}\n","  for label, confidence, bbox in detections:\n","    left, top, right, bottom = bbox2points(bbox)\n","    left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)\n","    cv2.rectangle(img, (left, top), (right, bottom), class_colors[label], 2)\n","    cv2.putText(img, \"{} [{:.2f}]\".format(label, float(confidence)),(left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,class_colors[label], 2)\n","    l.append(label)\n","  for j in l:\n","    d[j]=l.count(j)\n","  m1=list(d.keys())\n","  m2=list(d.values())\n","  zz=[\"There are\"]\n","  for z in range(len(m1)):\n","    zz.append(str(m2[z]))\n","    zz.append(m1[z])\n","  zz.append(\"before you\")\n","  global ccc\n","  if len(zz)==2:\n","    ccc=\"please try again, i cant detect any objects before you\"\n","  else:\n","    ccc=str(zz)\n","  cv2.imwrite(filename, img)\n","  return filename"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"kVBfU_k_692n","colab":{"base_uri":"https://localhost:8080/","height":519},"outputId":"d851a0b1-c6f7-46ff-b514-ecbafd4563f2"},"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.Javascript object>"],"application/javascript":["\n","    async function takePhoto(quality) {\n","      const div = document.createElement('div');\n","      const capture = document.createElement('button');\n","      capture.textContent = 'Capture';\n","      div.appendChild(capture);\n","      const video = document.createElement('video');\n","      video.style.display = 'block';\n","      const stream = await navigator.mediaDevices.getUserMedia({video: true});\n","      document.body.appendChild(div);\n","      div.appendChild(video);\n","      video.srcObject = stream;\n","      await video.play();\n","      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);\n","      await new Promise((resolve) => capture.onclick = resolve);\n","      const canvas = document.createElement('canvas');\n","      canvas.width = video.videoWidth;\n","      canvas.height = video.videoHeight;\n","      canvas.getContext('2d').drawImage(video, 0, 0);\n","      stream\n","      .getVideoTracks()[0].stop();\n","      div.remove();\n","      return canvas.toDataURL('image/jpeg', quality);\n","    }\n","    "]},"metadata":{}}],"source":["from IPython.display import Image\n","try:\n","  filename = take_photo()\n","  print('Saved to {}'.format(filename))\n","  \n","  # Show the image which was just taken.\n","  display(Image(filename))\n","except Exception as err:\n","  # Errors will be thrown if the user does not have a webcam or if they do not\n","  # grant the page permission to access it.\n","  print(str(err))\n","### NPL\n","\n","tts = gTTS(ccc) #Provide the string to convert to speech\n","tts.save('1.wav') #save the string converted to speech as a .wav file\n","sound_file = '1.wav'\n","Audio(sound_file, autoplay=True)"]},{"cell_type":"markdown","metadata":{"id":"_AltmMAr_l1X"},"source":["# LIVE CAM"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Rdd2RBwooApA"},"outputs":[],"source":["# JavaScript to properly create our live video stream using our webcam as input\n","def video_stream():\n","  js = Javascript('''\n","    var video;\n","    var div = null;\n","    var stream;\n","    var captureCanvas;\n","    var imgElement;\n","    var labelElement;\n","    \n","    var pendingResolve = null;\n","    var shutdown = false;\n","    \n","    function removeDom() {\n","       stream.getVideoTracks()[0].stop();\n","       video.remove();\n","       div.remove();\n","       video = null;\n","       div = null;\n","       stream = null;\n","       imgElement = null;\n","       captureCanvas = null;\n","       labelElement = null;\n","    }\n","    \n","    function onAnimationFrame() {\n","      if (!shutdown) {\n","        window.requestAnimationFrame(onAnimationFrame);\n","      }\n","      if (pendingResolve) {\n","        var result = \"\";\n","        if (!shutdown) {\n","          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);\n","          result = captureCanvas.toDataURL('image/jpeg', 0.8)\n","        }\n","        var lp = pendingResolve;\n","        pendingResolve = null;\n","        lp(result);\n","      }\n","    }\n","    \n","    async function createDom() {\n","      if (div !== null) {\n","        return stream;\n","      }\n","\n","      div = document.createElement('div');\n","      div.style.border = '2px solid black';\n","      div.style.padding = '3px';\n","      div.style.width = '100%';\n","      div.style.maxWidth = '600px';\n","      document.body.appendChild(div);\n","      \n","      const modelOut = document.createElement('div');\n","      modelOut.innerHTML = \"<span>Status:</span>\";\n","      labelElement = document.createElement('span');\n","      labelElement.innerText = 'No data';\n","      labelElement.style.fontWeight = 'bold';\n","      modelOut.appendChild(labelElement);\n","      div.appendChild(modelOut);\n","           \n","      video = document.createElement('video');\n","      video.style.display = 'block';\n","      video.width = div.clientWidth - 6;\n","      video.setAttribute('playsinline', '');\n","      video.onclick = () => { shutdown = true; };\n","      stream = await navigator.mediaDevices.getUserMedia(\n","          {video: { facingMode: \"environment\"}});\n","      div.appendChild(video);\n","\n","      imgElement = document.createElement('img');\n","      imgElement.style.position = 'absolute';\n","      imgElement.style.zIndex = 1;\n","      imgElement.onclick = () => { shutdown = true; };\n","      div.appendChild(imgElement);\n","      \n","      const instruction = document.createElement('div');\n","      instruction.innerHTML = \n","          '<span style=\"color: red; font-weight: bold;\">' +\n","          'When finished, click here or on the video to stop this demo</span>';\n","      div.appendChild(instruction);\n","      instruction.onclick = () => { shutdown = true; };\n","      \n","      video.srcObject = stream;\n","      await video.play();\n","\n","      captureCanvas = document.createElement('canvas');\n","      captureCanvas.width = 640; //video.videoWidth;\n","      captureCanvas.height = 480; //video.videoHeight;\n","      window.requestAnimationFrame(onAnimationFrame);\n","      \n","      return stream;\n","    }\n","    async function stream_frame(label, imgData) {\n","      if (shutdown) {\n","        removeDom();\n","        shutdown = false;\n","        return '';\n","      }\n","\n","      var preCreate = Date.now();\n","      stream = await createDom();\n","      \n","      var preShow = Date.now();\n","      if (label != \"\") {\n","        labelElement.innerHTML = label;\n","      }\n","            \n","      if (imgData != \"\") {\n","        var videoRect = video.getClientRects()[0];\n","        imgElement.style.top = videoRect.top + \"px\";\n","        imgElement.style.left = videoRect.left + \"px\";\n","        imgElement.style.width = videoRect.width + \"px\";\n","        imgElement.style.height = videoRect.height + \"px\";\n","        imgElement.src = imgData;\n","      }\n","      \n","      var preCapture = Date.now();\n","      var result = await new Promise(function(resolve, reject) {\n","        pendingResolve = resolve;\n","      });\n","      shutdown = false;\n","      \n","      return {'create': preShow - preCreate, \n","              'show': preCapture - preShow, \n","              'capture': Date.now() - preCapture,\n","              'img': result};\n","    }\n","    ''')\n","\n","  display(js)\n","  \n","def video_frame(label, bbox):\n","  data = eval_js('stream_frame(\"{}\", \"{}\")'.format(label, bbox))\n","  return data"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":380},"executionInfo":{"elapsed":393,"status":"error","timestamp":1653290144485,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"},"user_tz":-330},"id":"ZlKArrzsyZ_Y","outputId":"6cb091dd-8bf3-469a-a760-b2788da67f82"},"outputs":[{"output_type":"error","ename":"MessageError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mMessageError\u001b[0m                              Traceback (most recent call last)","\u001b[0;32m<ipython-input-13-13ebdab08e18>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mjs_reply\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvideo_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel_html\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mjs_reply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-12-a05efe86d862>\u001b[0m in \u001b[0;36mvideo_frame\u001b[0;34m(label, bbox)\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mvideo_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m   \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval_js\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'stream_frame(\"{}\", \"{}\")'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/output/_js.py\u001b[0m in \u001b[0;36meval_js\u001b[0;34m(script, ignore_result, timeout_sec)\u001b[0m\n\u001b[1;32m     38\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mignore_result\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m     \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0m_message\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_reply_from_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mread_reply_from_input\u001b[0;34m(message_id, timeout_sec)\u001b[0m\n\u001b[1;32m    104\u001b[0m         reply.get('colab_msg_id') == message_id):\n\u001b[1;32m    105\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;34m'error'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mMessageError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'error'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mMessageError\u001b[0m: Error: Cell has no view"]}],"source":["# start streaming video from webcam\n","video_stream()\n","# label for video\n","label_html = 'Capturing...'\n","# initialze bounding box to empty\n","bbox = ''\n","count = 0 \n","while True:\n","    js_reply = video_frame(label_html, bbox)\n","    if not js_reply:\n","        break\n","\n","    # convert JS response to OpenCV Image\n","    frame = js_to_image(js_reply[\"img\"])\n","\n","    # create transparent overlay for bounding box\n","    bbox_array = np.zeros([480,640,4], dtype=np.uint8)\n","\n","    # call our darknet helper on video frame\n","    detections, width_ratio, height_ratio = darknet_helper(frame, width, height)\n","    count=0\n","    # loop through detections and draw them on transparent overlay image\n","    for label, confidence, bbox in detections:\n","      count+=1\n","      left, top, right, bottom = bbox2points(bbox)\n","      left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)\n","      bbox_array = cv2.rectangle(bbox_array, (left, top), (right, bottom), class_colors[label], 2)\n","      bbox_array = cv2.putText(bbox_array, \"{} [{:.2f}]\".format(label, float(confidence)),\n","                        (left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,\n","                        class_colors[label], 2)\n","\n","    bbox_array[:,:,3] = (bbox_array.max(axis = 2) > 0 ).astype(int) * 255\n","    # convert overlay of bbox into bytes\n","    bbox_bytes = bbox_to_bytes(bbox_array)\n","    # update bbox so next frame gets new overlay\n","    bbox = bbox_bytes"]},{"cell_type":"markdown","source":["# Voice"],"metadata":{"id":"bzIMNlacjEk_"}},{"cell_type":"code","source":["!pip install pyttsx3\n","!pip install SpeechRecognition\n","!pip install gTTS\n","!pip install transformers command\n","!pip install pipwin\n","!pip install conda"],"metadata":{"id":"ATbWbkWgkJCF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!apt install libasound2-dev portaudio19-dev libportaudio2 libportaudiocpp0 ffmpeg\n","!pip install PyAudio"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_TX-jTakmQbN","executionInfo":{"status":"ok","timestamp":1652851260553,"user_tz":-330,"elapsed":13843,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}},"outputId":"42747d1b-c626-47b3-d7d4-7a13f6175a3f"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Reading package lists... Done\n","Building dependency tree       \n","Reading state information... Done\n","libasound2-dev is already the newest version (1.1.3-5ubuntu0.6).\n","ffmpeg is already the newest version (7:3.4.8-0ubuntu0.2).\n","The following packages were automatically installed and are no longer required:\n","  libnvidia-common-460 nsight-compute-2020.2.0\n","Use 'apt autoremove' to remove them.\n","Suggested packages:\n","  portaudio19-doc\n","The following NEW packages will be installed:\n","  libportaudio2 libportaudiocpp0 portaudio19-dev\n","0 upgraded, 3 newly installed, 0 to remove and 67 not upgraded.\n","Need to get 184 kB of archives.\n","After this operation, 891 kB of additional disk space will be used.\n","Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libportaudio2 amd64 19.6.0-1 [64.6 kB]\n","Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libportaudiocpp0 amd64 19.6.0-1 [15.1 kB]\n","Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 portaudio19-dev amd64 19.6.0-1 [104 kB]\n","Fetched 184 kB in 1s (320 kB/s)\n","Selecting previously unselected package libportaudio2:amd64.\n","(Reading database ... 155203 files and directories currently installed.)\n","Preparing to unpack .../libportaudio2_19.6.0-1_amd64.deb ...\n","Unpacking libportaudio2:amd64 (19.6.0-1) ...\n","Selecting previously unselected package libportaudiocpp0:amd64.\n","Preparing to unpack .../libportaudiocpp0_19.6.0-1_amd64.deb ...\n","Unpacking libportaudiocpp0:amd64 (19.6.0-1) ...\n","Selecting previously unselected package portaudio19-dev:amd64.\n","Preparing to unpack .../portaudio19-dev_19.6.0-1_amd64.deb ...\n","Unpacking portaudio19-dev:amd64 (19.6.0-1) ...\n","Setting up libportaudio2:amd64 (19.6.0-1) ...\n","Setting up libportaudiocpp0:amd64 (19.6.0-1) ...\n","Setting up portaudio19-dev:amd64 (19.6.0-1) ...\n","Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n","/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n","\n","Collecting PyAudio\n","  Using cached PyAudio-0.2.11.tar.gz (37 kB)\n","Building wheels for collected packages: PyAudio\n","  Building wheel for PyAudio (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for PyAudio: filename=PyAudio-0.2.11-cp37-cp37m-linux_x86_64.whl size=52616 sha256=c17b9528bf8f3cef36d0d010d1d84b7f8069435c866ac7d8c689adec6dbab9e2\n","  Stored in directory: /root/.cache/pip/wheels/40/2e/4c/b71e7e96c861a46e6213bc6bb482b94dcf293a92c5e736c1ec\n","Successfully built PyAudio\n","Installing collected packages: PyAudio\n","Successfully installed PyAudio-0.2.11\n"]}]},{"cell_type":"code","source":["###############################################################################\n","#                             CHATBOT                                         #\n","###############################################################################\n","# Setup\n","## for speech-to-text\n","import speech_recognition as sr\n","\n","## for text-to-speech\n","from gtts import gTTS\n","\n","## for language model\n","import transformers\n","\n","## for data\n","import os\n","import datetime\n","import numpy as np\n","\n","#colab\n","from io import BytesIO\n","from base64 import b64decode\n","from google.colab import output\n","from IPython.display import Javascript\n","\n","\n","# Build the AI\n","class ChatBot():\n","    def __init__(self, name):\n","        print(\"--- starting up\", name, \"---\")\n","        self.name = name\n","\n","    def speech_to_text(self):\n","        recognizer = sr.Recognizer()\n","        \n","        RECORD = \"\"\"\n","        const sleep  = time => new Promise(resolve => setTimeout(resolve, time))\n","        const b2text = blob => new Promise(resolve => {\n","          const reader = new FileReader()\n","          reader.onloadend = e => resolve(e.srcElement.result)\n","          reader.readAsDataURL(blob)\n","        })\n","        var record = time => new Promise(async resolve => {\n","          stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n","          recorder = new MediaRecorder(stream)\n","          chunks = []\n","          recorder.ondataavailable = e => chunks.push(e.data)\n","          recorder.start()\n","          await sleep(time)\n","          recorder.onstop = async ()=>{\n","            blob = new Blob(chunks)\n","            text = await b2text(blob)\n","            resolve(text)\n","          }\n","          recorder.stop()\n","        })\n","        \"\"\"\n","\n","        def record(sec=3):\n","          print(\"Speak Now...\")\n","          display(Javascript(RECORD))\n","          sec += 1\n","          s = output.eval_js('record(%d)' % (sec*1000))\n","          print(\"Done Recording !\")\n","          b = b64decode(s.split(',')[1])\n","          return b #byte stream\n","\n","\n","\n","        with sr.Microphone() as mic:\n","            print(\"listening...\")\n","            audio = recognizer.listen(mic)\n","        try:\n","            self.text = recognizer.recognize_google(audio)\n","            print(\"me --> \", self.text)\n","        except:\n","            print(\"me -->  ERROR\")\n","\n","    @staticmethod\n","    def text_to_speech(text):\n","        print(\"ai --> \", text)\n","        speaker = gTTS(text=text, lang=\"en\", slow=False)\n","        speaker.save(\"res.mp3\")\n","        os.system(\"afplay res.mp3\")  #mac->afplay | windows->start\n","        os.remove(\"res.mp3\")\n","\n","    def wake_up(self, text):\n","        return True if self.name in text.lower() else False\n","\n","    @staticmethod\n","    def action_time():\n","        return datetime.datetime.now().time().strftime('%H:%M')\n","\n","\n","# Run the AI\n","if __name__ == \"__main__\":\n","    \n","    ai = ChatBot(name=\"maya\")\n","    nlp = transformers.pipeline(\"conversational\", model=\"microsoft/DialoGPT-medium\")\n","    os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n","\n","    while True:\n","        ai.speech_to_text()\n","\n","        ## wake up\n","        if ai.wake_up(ai.text) is True:\n","            res = \"Hello I am Maya the AI, what can I do for you?\"\n","        \n","        ## action time\n","        elif \"time\" in ai.text:\n","            res = ai.action_time()\n","        \n","        ## respond politely\n","        elif any(i in ai.text for i in [\"thank\",\"thanks\"]):\n","            res = np.random.choice([\"you're welcome!\",\"anytime!\",\"no problem!\",\"cool!\",\"I'm here if you need me!\",\"peace out!\"])\n","        \n","        ## conversation\n","        else:   \n","            chat = nlp(transformers.Conversation(ai.text), pad_token_id=50256)\n","            res = str(chat)\n","            res = res[res.find(\"bot >> \")+6:].strip()\n","\n","        ai.text_to_speech(res)"],"metadata":{"id":"OUVTg7h8j8uF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from io import BytesIO\n","from base64 import b64decode\n","from google.colab import output\n","from IPython.display import Javascript\n","# all imports\n","from io import BytesIO\n","from base64 import b64decode\n","from google.colab import output\n","from IPython.display import Javascript\n","\n","RECORD = \"\"\"\n","const sleep  = time => new Promise(resolve => setTimeout(resolve, time))\n","const b2text = blob => new Promise(resolve => {\n","  const reader = new FileReader()\n","  reader.onloadend = e => resolve(e.srcElement.result)\n","  reader.readAsDataURL(blob)\n","})\n","var record = time => new Promise(async resolve => {\n","  stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n","  recorder = new MediaRecorder(stream)\n","  chunks = []\n","  recorder.ondataavailable = e => chunks.push(e.data)\n","  recorder.start()\n","  await sleep(time)\n","  recorder.onstop = async ()=>{\n","    blob = new Blob(chunks)\n","    text = await b2text(blob)\n","    resolve(text)\n","  }\n","  recorder.stop()\n","})\n","\"\"\"\n","\n","def record(sec=3):\n","  print(\"Speak Now...\")\n","  display(Javascript(RECORD))\n","  sec += 1\n","  s = output.eval_js('record(%d)' % (sec*1000))\n","  print(\"Done Recording !\")\n","  b = b64decode(s.split(',')[1])\n","  return b #byte stream\n","\n","import speech_recognition as sr\n","r = sr.Recognizer()\n","with sr.Microphone() as source:\n","    print(\"Speak Anything :\")\n","    audio = r.listen(source)\n","    try:\n","        text = r.recognize_google(audio)\n","        print(\"You said : {}\".format(text))\n","    except:\n","        print(\"Sorry could not recognize what you said\")"],"metadata":{"id":"HjSkv8tepGqk","executionInfo":{"status":"error","timestamp":1652851935483,"user_tz":-330,"elapsed":1667,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}},"outputId":"e19698a2-5970-446b-b446-2db4463211a5","colab":{"base_uri":"https://localhost:8080/","height":380}},"execution_count":null,"outputs":[{"output_type":"error","ename":"OSError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)","\u001b[0;32m<ipython-input-22-20b0cba54e90>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mspeech_recognition\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRecognizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0msr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMicrophone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Speak Anything :\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0maudio\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlisten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/speech_recognition/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, device_index, sample_rate, chunk_size)\u001b[0m\n\u001b[1;32m     84\u001b[0m                 \u001b[0;32massert\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mdevice_index\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Device index out of range ({} devices available; device index should be between 0 and {} inclusive)\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcount\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0msample_rate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# automatically set the sample rate to the hardware's default sample rate if not specified\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m                 \u001b[0mdevice_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maudio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_device_info_by_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_index\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdevice_index\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0maudio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_input_device_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m                 \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_info\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"defaultSampleRate\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mdevice_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"defaultSampleRate\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Invalid device info returned from PyAudio: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_info\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m                 \u001b[0msample_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"defaultSampleRate\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyaudio.py\u001b[0m in \u001b[0;36mget_default_input_device_info\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    947\u001b[0m         \"\"\"\n\u001b[1;32m    948\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 949\u001b[0;31m         \u001b[0mdevice_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_default_input_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    950\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_device_info_by_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    951\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mOSError\u001b[0m: No Default Input Device Available"]}]},{"cell_type":"code","source":["import pyttsx3 #pip install pyttsx3\n","import speech_recognition as sr     #pip install speechRecognition\n","import datetime\n","import wikipedia #pip install wikipedia\n","import webbrowser\n","import os\n","import smtplib\n","import screen_brightness_control as sbc\n","import wolframalpha\n","import json\n","import requests\n","import subprocess\n","import time\n","\n","\n","engine = pyttsx3.init('sapi5')\n","voices = engine.getProperty('voices')\n","# print(voices[1].id)\n","engine.setProperty('voice', voices[2].id)\n","\n","\n","def speak(audio):\n","    engine.say(audio)\n","    engine.runAndWait()\n","\n","\n","def wishMe():\n","    hour = int(datetime.datetime.now().hour)\n","    if hour>=0 and hour<12:\n","        speak(\"Good Morning!\")\n","\n","    elif hour>=12 and hour<18:\n","        speak(\"Good Afternoon!\")   \n","\n","    else:\n","        speak(\"Good Evening!\")  \n","\n","    speak(\" I am your assistant. How may I help you ?? \")       \n","\n","def takeCommand():\n","    #It takes microphone input from the user and returns string output\n","\n","    r = sr.Recognizer()\n","    with sr.Microphone() as source:\n","        print(\"Listening...\")\n","        r.pause_threshold = 0.7\n","        audio = r.listen(source)\n","\n","    try:\n","        print(\"Recognizing...\")    \n","        query = r.recognize_google(audio, language='en-in')\n","        print(f\"User said: {query}\\n\")\n","\n","    except Exception as e:\n","        # print(e)    \n","        print(\"Say that again please...\")  \n","        return \"None\"\n","    return query\n","\n","def sendEmail(to, content):\n","    server = smtplib.SMTP('smtp.gmail.com', 587)\n","    server.ehlo()\n","    server.starttls()\n","    server.login('shakthiadityamanivannan@gmail.com', 'ktmrc690')\n","    server.sendmail('shakthiadityamanivannan@gmail.com', to, content)\n","    server.close()\n","\n","if __name__ == \"__main__\":\n","    wishMe()\n","    while True:\n","    # if 1:\n","        query = takeCommand().lower()\n","\n","        # Logic for executing tasks based on query\n","        if 'wikipedia' in query:\n","            speak('Searching Wikipedia...')\n","            query = query.replace(\"wikipedia\", \"\")\n","            results = wikipedia.summary(query, sentences=2)\n","            speak(\"According to Wikipedia\")\n","            print(results)\n","            speak(results)\n","\n","        elif 'open youtube' in query:\n","            webbrowser.open(\"youtube.com\")\n","\n","        elif 'open google' in query:\n","            webbrowser.open(\"google.com\")\n","\n","        elif 'open stack overflow' in query:\n","            webbrowser.open(\"stackoverflow.com\")   \n","\n","        elif 'play music' in query:\n","            music_dir = 'D:\\Music'\n","            songs = os.listdir(music_dir)\n","            print(songs)    \n","            os.startfile(os.path.join(music_dir, songs[0]))\n","\n","        elif 'the time' in query:\n","            strTime = datetime.datetime.now().strftime(\"%H:%M:%S\")    \n","            speak(f\"Sir, the time is {strTime}\")\n","\n","        elif 'shutdown' in query:\n","            speak(\"Shutting down laptop\")\n","            os.system(\"shutdown /s /t 1\")\n","\n","        elif 'email to me' in query:\n","            try:\n","                speak(\"What should I say?\")\n","                content = takeCommand()\n","                to = \"shakthiadityamanivannan@gmail.com\"    \n","                sendEmail(to, content)\n","                speak(\"Email has been sent!\")\n","            except Exception as e:\n","                print(e)\n","                speak(\"Sorry Sir. I am not able to send this email\")\n","\n","        elif \"current brightness\" in query:\n","            current_brightness = sbc.get_brightness()\n","            print(f\"Current brightness of your device is {current_brightness}\")\n","            speak(f\"Current brightness of your device is {current_brightness}\")\n","\n","        elif \"adjust brightness\" in query:\n","            speak(\"What value should brightness be adjusted?\")\n","            desired_brightness_1 = takeCommand()\n","            print(desired_brightness_1)\n","            desired_brightness_2 = takeCommand()\n","            print(desired_brightness_2)\n","            desired_brightness = desired_brightness_1 + desired_brightness_2\n","            speak(f\"Current brightness adjusted to {desired_brightness}\")\n","            print(f\"Current brightness adjusted to {desired_brightness}\")\n","            sbc.set_brightness(desired_brightness,display = None,verbose_error = False,no_return = False)\n","\n","        elif \"weather\" in query:\n","            api_key=\"8ef61edcf1c576d65d836254e11ea420\"\n","            base_url=\"https://api.openweathermap.org/data/2.5/weather?\"\n","            speak(\"whats the city name\")\n","            city_name = takeCommand()\n","            complete_url = base_url+\"appid=\"+api_key+\"&q=\"+city_name\n","            response = requests.get(complete_url)\n","            x=response.json()\n","            if x[\"cod\"]!=\"404\":\n","                y=x[\"main\"]\n","                current_temperature = y[\"temp\"]\n","                current_humidiy = y[\"humidity\"]\n","                z = x[\"weather\"]\n","                weather_description = z[0][\"description\"]\n","                speak(\" Temperature in kelvin unit is \" +\n","                      str(current_temperature) +\n","                      \"\\n humidity in percentage is \" +\n","                      str(current_humidiy) +\n","                      \"\\n description  \" +\n","                      str(weather_description))\n","                print(\" Temperature in kelvin unit = \" +\n","                      str(current_temperature) +\n","                      \"\\n humidity (in percentage) = \" +\n","                      str(current_humidiy) +\n","                      \"\\n description = \" +\n","                      str(weather_description))\n","\n","            else:\n","                speak(\" City Not Found \")\n","\n","        elif 'search'  in query:\n","            query = query.replace(\"search\", \"\")\n","            webbrowser.open_new_tab(query)\n","            time.sleep(0.8)\n","\n","        elif 'who are you' in query or 'what can you do' in query:\n","            speak('I am G-one version 1 point O your persoanl assistant. I am programmed to minor tasks like'\n","                  'opening youtube,google chrome,gmail and stackoverflow ,predict time,take a photo,search wikipedia,predict weather' \n","                  'in different cities , get top headline news from times of india and you can ask me computational or geographical questions too!')\n","\n","        elif \"who made you\" in query or \"who created you\" in query:\n","            speak(\"I was built by AI Lab\")\n","            print(\"I was built by AI Lab\")\n","\n","        elif 'news' in query:\n","            try:\n","                speak(\"Which news article would you like to read ??\")\n","                print(\"Which news article would you like to read ??\")\n","                article = takeCommand()\n","\n","                if \"Times of india\" in article:\n","                    webbrowser.open_new_tab(\"https://timesofindia.indiatimes.com/home/headlines\")\n","                    speak('Here are some headlines from the Times of India,Happy reading')\n","                    time.sleep(0.8)\n","                elif \"The hindu\" in article:\n","                    news = webbrowser.open_new_tab(\"https://www.thehindu.com/\")\n","                    speak('Here are some headlines from the hindu,Happy reading')\n","            except:\n","                speak(\"The news article that you asked for is not available\")\n","                print(\"The news article that you asked for is not available\")\n","\n","            \n","            \n"," \n","\n","        # elif \"list of monitors\" in query:\n","        #     monitors = sbc.list_monitors()\n","        #     speak(f\"List of monitors are {monitors}\")\n","        #     print(monitors)\n","\n","        elif 'stop listening' in query:\n","            speak(\"Okay sir. Thank you\")\n","            exit()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":373},"id":"2y53XeesjILS","executionInfo":{"status":"error","timestamp":1652850440629,"user_tz":-330,"elapsed":22,"user":{"displayName":"Cpluz Shrijayan","userId":"04166341969368347220"}},"outputId":"d0dd735e-12e4-4f02-9896-e5b1d491f416"},"execution_count":null,"outputs":[{"output_type":"error","ename":"ModuleNotFoundError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m<ipython-input-2-382e3b807ac4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpyttsx3\u001b[0m \u001b[0;31m#pip install pyttsx3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mspeech_recognition\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msr\u001b[0m     \u001b[0;31m#pip install speechRecognition\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mwikipedia\u001b[0m \u001b[0;31m#pip install wikipedia\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mwebbrowser\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pyttsx3'","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}]},{"cell_type":"markdown","metadata":{"id":"QBDJZsZBbe-v"},"source":["#Face Recognition "]},{"cell_type":"code","source":["import face_recognition\n","\n","picture_of_me = face_recognition.load_image_file(\"/content/WIN_20220517_15_08_29_Pro.jpg\")\n","my_face_encoding = face_recognition.face_encodings(picture_of_me)[0]\n","\n","# my_face_encoding now contains a universal 'encoding' of my facial features that can be compared to any other picture of a face!\n","\n","unknown_picture = face_recognition.load_image_file(\"/content/WIN_20220517_15_13_25_Pro.jpg\")\n","unknown_face_encoding = face_recognition.face_encodings(unknown_picture)[0]\n","\n","# Now we can see the two face encodings are of the same person with `compare_faces`!\n","\n","results = face_recognition.compare_faces([my_face_encoding], unknown_face_encoding)\n","\n","if results[0] == True:\n","    print(\"It's a picture of abi!\")\n","else:\n","    print(\"It's not a picture of me!\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bKlS-ucmXlRY","executionInfo":{"status":"ok","timestamp":1652780693117,"user_tz":-330,"elapsed":2315,"user":{"displayName":"SHRI JAYAN R A","userId":"07758452110844830688"}},"outputId":"013bff93-fe4c-441e-b968-7f06d226a648"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["It's a picture of abi!\n"]}]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":["m2qjB8dJnTCb","rSNhYIf4cTUk","j7c4QGSDDJrD","QBDJZsZBbe-v"],"provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}