import IPython #from IPython.display import display, Javascript #from PIL import Image import google #from google.colab import output #from google.colab.output import eval_js import numpy as np import cv2 import io #from io import BytesIO import base64 class WebCam(): def __init__(self): self.out_img = None self.enc_img = None self.img_str = None output.register_callback('notebook.run', WebCam.capture) @staticmethod def capture(img_str): #decode to image decimg = base64.b64decode(self.img_str.split(',')[1], validate=True) decimg = Image.open(BytesIO(decimg)) decimg = np.array(decimg, dtype=np.uint8); decimg = cv2.cvtColor(decimg, cv2.COLOR_BGR2RGB) #############your process############### out_img = cv2.Canny(decimg,100,200) #out_img = decimg #############your process############### #encode to string _, encimg = cv2.imencode(".jpg", out_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80]) img_str2 = encimg.tobytes() img_str2 = "data:image/jpeg;base64," + base64.b64encode(img_str2).decode('utf-8') self.out_img self.enc_img = encimg self.img_str = img_str2 return IPython.display.JSON({'img_str': img_str2}) @staticmethod def use_cam(quality=0.8, display_size=500): js = Javascript(''' async function useCam(quality) { const div = document.createElement('div'); document.body.appendChild(div); //video element const video = document.createElement('video'); video.style.display = 'None'; const stream = await navigator.mediaDevices.getUserMedia({video: true}); div.appendChild(video); video.srcObject = stream; await video.play(); //canvas for display. frame rate is depending on display size and jpeg quality. const src_canvas = document.createElement('canvas'); src_canvas.width = display_size; src_canvas.height = display_size * video.videoHeight / video.videoWidth; const src_canvasCtx = src_canvas.getContext('2d'); src_canvasCtx.translate(src_canvas.width, 0); src_canvasCtx.scale(-1, 1); div.appendChild(src_canvas); const dst_canvas = document.createElement('canvas'); dst_canvas.width = src_canvas.width; dst_canvas.height = src_canvas.height; const dst_canvasCtx = dst_canvas.getContext('2d'); div.appendChild(dst_canvas); //exit button const btn_div = document.createElement('div'); document.body.appendChild(btn_div); const exit_btn = document.createElement('button'); exit_btn.textContent = 'Exit'; var exit_flg = true exit_btn.onclick = function() {exit_flg = false}; btn_div.appendChild(exit_btn); // Resize the output to fit the video element. google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true); var send_num = 0 // loop _canvasUpdate(); async function _canvasUpdate() { src_canvasCtx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, src_canvas.width, src_canvas.height); if (send_num<1){ send_num += 1 const img = src_canvas.toDataURL('image/jpeg', quality); const result = google.colab.kernel.invokeFunction('notebook.run', [img], {}); result.then(function(value) { parse = JSON.parse(JSON.stringify(value))["data"] parse = JSON.parse(JSON.stringify(parse))["application/json"] parse = JSON.parse(JSON.stringify(parse))["img_str"] var image = new Image() image.src = parse; image.onload = function(){dst_canvasCtx.drawImage(image, 0, 0)} send_num -= 1 }) } if (exit_flg){ requestAnimationFrame(_canvasUpdate); }else{ stream.getVideoTracks()[0].stop(); } }; } ''') IPython.display(js) data = google.colab.output.eval_js(f'useCam({quality})') def run(img_str): #decode to image decimg = base64.b64decode(img_str.split(',')[1], validate=True) decimg = Image.open(BytesIO(decimg)) decimg = np.array(decimg, dtype=np.uint8); decimg = cv2.cvtColor(decimg, cv2.COLOR_BGR2RGB) #############your process############### out_img = cv2.Canny(decimg,100,200) #out_img = decimg #############your process############### #encode to string _, encimg = cv2.imencode(".jpg", out_img, [int(cv2.IMWRITE_JPEG_QUALITY), 80]) img_str = encimg.tobytes() img_str = "data:image/jpeg;base64," + base64.b64encode(img_str).decode('utf-8') return IPython.display.JSON({'img_str': img_str}) def use_cam(quality=0.8): js = Javascript(''' async function useCam(quality) { const div = document.createElement('div'); document.body.appendChild(div); //video element const video = document.createElement('video'); video.style.display = 'None'; const stream = await navigator.mediaDevices.getUserMedia({video: true}); div.appendChild(video); video.srcObject = stream; await video.play(); //canvas for display. frame rate is depending on display size and jpeg quality. display_size = 500 const src_canvas = document.createElement('canvas'); src_canvas.width = display_size; src_canvas.height = display_size * video.videoHeight / video.videoWidth; const src_canvasCtx = src_canvas.getContext('2d'); src_canvasCtx.translate(src_canvas.width, 0); src_canvasCtx.scale(-1, 1); div.appendChild(src_canvas); const dst_canvas = document.createElement('canvas'); dst_canvas.width = src_canvas.width; dst_canvas.height = src_canvas.height; const dst_canvasCtx = dst_canvas.getContext('2d'); div.appendChild(dst_canvas); //exit button const btn_div = document.createElement('div'); document.body.appendChild(btn_div); const exit_btn = document.createElement('button'); exit_btn.textContent = 'Exit'; var exit_flg = true exit_btn.onclick = function() {exit_flg = false}; btn_div.appendChild(exit_btn); // Resize the output to fit the video element. google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true); var send_num = 0 // loop _canvasUpdate(); async function _canvasUpdate() { src_canvasCtx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, src_canvas.width, src_canvas.height); if (send_num<1){ send_num += 1 const img = src_canvas.toDataURL('image/jpeg', quality); const result = google.colab.kernel.invokeFunction('notebook.run', [img], {}); result.then(function(value) { parse = JSON.parse(JSON.stringify(value))["data"] parse = JSON.parse(JSON.stringify(parse))["application/json"] parse = JSON.parse(JSON.stringify(parse))["img_str"] var image = new Image() image.src = parse; image.onload = function(){dst_canvasCtx.drawImage(image, 0, 0)} send_num -= 1 }) } if (exit_flg){ requestAnimationFrame(_canvasUpdate); }else{ stream.getVideoTracks()[0].stop(); } }; } ''') display(js) data = eval_js('useCam({})'.format(quality)) # 実行する。 #use_cam()