import { HttpClient } from "@angular/common/http";
import { environment } from "../../environments/environment";
import { Observable } from "rxjs";
import * as tf from "@tensorflow/tfjs";
import { getSupportedInputTypes } from "@angular/cdk/platform";
import { AppUtils } from "../app-utils";
// const { JSDOM } = require('jsdom');
// import cv from 'opencv.js';
declare var cv;
declare var window: { innerWidth: any; innerHeight: any };
export class Drawing {
private canvas: HTMLCanvasElement;
private context: CanvasRenderingContext2D;
private paint: boolean;
private clickX: number[] = [];
private clickY: number[] = [];
private clickDrag: boolean[] = [];
public environment: { production: boolean; activityVersion: number; kidaia: boolean };
model: any;
predictions: any;
canvas2: HTMLCanvasElement;
ctx: CanvasRenderingContext2D;
public onlineRecognitionPromise: Observable<any>;
public onlineRecognitionResult: Object;
constructor(private http: HttpClient) {
this.environment = environment;
this.canvas = document.getElementById("canvas") as unknown as HTMLCanvasElement;
this.canvas2 = document.createElement("canvas") as HTMLCanvasElement;
this.ctx = this.canvas2.getContext("2d");
// document.querySelector('.canvaswrapper').append(this.canvas2);
this.canvas.width = 500;
this.canvas.height = 500;
this.setCanvasSize();
this.setCanvasSize();
this.configureContext();
this.clearCanvas();
this.redraw();
this.createUserEvents();
this.loadModel();
}
async loadModel() {
this.model = await tf.loadLayersModel("/assets/tfjs/model.json");
console.log(this.model);
}
/*
async predict(imageData) {
const pred = await tf.tidy(() => {
// Convert the canvas pixels to
let img = tf.browser.fromPixels(imageData).resizeNearestNeighbor([20, 20]).mean(2).expandDims(2).expandDims().toFloat();
img = img.div(255.0);
// Make and format the predications
const output = this.model.predict(img) as any;
// Save predictions on the component
this.predictions = Array.from(output.dataSync());
});
}
*/
// pour comparer les coordonnées x des rectangles
compare(a, b) {
if (a.x > b.x) return 1;
if (b.x > a.x) return -1;
return 0;
}
readImage(canvas, maxval = 255) {
const src = cv.imread(canvas);
const gray = new cv.Mat();
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY);
cv.threshold(gray, gray, 0, maxval, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
src.delete();
return gray;
}
/**
* OpenCV find contours
*/
readContour(image) {
const contours = new cv.MatVector();
const hierarchy = new cv.Mat();
cv.findContours(image, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE);
console.log("contours size ", contours.size());
const rectContours = new Array();
for (let i = 0; i < contours.size(); i++) {
// You can try more different parameters
const rect = cv.boundingRect(contours.get(i));
rectContours.push(rect);
}
contours.delete();
hierarchy.delete();
rectContours.sort(this.compare);
return rectContours;
}
tfjsTextRecognition(): Observable<string[]> {
return new Observable<string[]>(subscriber => {
const imgData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height);
const resultsPrediction = new Array();
const maxArray = new Array();
let result = "";
//
// change non-opaque pixels to white
//
const data = imgData.data;
for (let i = 0; i < data.length; i += 4) {
if (data[i + 3] < 255) {
data[i] = 255;
data[i + 1] = 255;
data[i + 2] = 255;
data[i + 3] = 255;
}
}
this.ctx.putImageData(imgData, 0, 0);
//
// lire l'image et extraire les contours
//
const imgDataURL = this.canvas2.toDataURL("image/jpeg");
let gray = new cv.Mat();
let gray1 = new cv.Mat();
gray = this.readImage(this.canvas2);
let rectContours = this.readContour(gray);
this.clearCanvas2();
// ################### Redessiner l'image pour le centrer ###########################
cv.threshold(gray, gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
let diff = 0;
let imageData1 = imgData;
rectContours.forEach(rect => {
const border = 0;
this.drawContours(rect);
const width = rect.width;
const height = rect.height;
console.log("x:", rect.x);
console.log("y:", rect.y);
console.log("gray cols ", gray.cols);
console.log("gray rows ", gray.rows);
console.log("canvas width ", this.canvas.width);
console.log("canvas height ", this.canvas.height);
console.log("width ", width);
console.log("height ", height);
rect = new cv.Rect(rect.x, rect.y, width, height);
gray1 = gray.roi(rect);
// console.log(imgData.data);
// ++++++ Dimensions de l'image à redessiner ++++++++++
let newWidth = width;
let newheight = height;
// ++++++++ Cas où l'image prend tout le canvas ++++++++++
if (width > gray.cols / 3) newWidth = gray.cols / 3;
if (height > gray.rows / 3) newheight = gray.rows / 3;
const dsize = new cv.Size(newWidth, newheight);
// ++++++++++ Cas où la hauteur est 2 fois plus grand que la largeur +++++++++
// if (newheight > (newWidth * 2))
// dsize = new cv.Size(newWidth, newWidth * 2);
cv.resize(gray1, gray1, dsize, 0, 0, cv.INTER_AREA);
cv.cvtColor(gray1, gray1, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(gray1.cols, gray1.rows);
imageData1.data.set(new Uint8ClampedArray(gray1.data, gray1.cols, gray1.rows));
this.ctx.putImageData(imageData1, this.canvas.width / 4 + diff, this.canvas.height / 4);
diff = diff + newWidth + 100;
});
// ################## Recupérer l'image centrée ########################
const imgData1 = this.ctx.getImageData(0, 0, this.canvas.width, this.canvas.height);
const data1 = imgData1.data;
// change non-opaque pixels to white
for (let i = 0; i < data1.length; i += 4) {
if (data1[i + 3] < 255) {
data1[i] = 255;
data1[i + 1] = 255;
data1[i + 2] = 255;
data1[i + 3] = 255;
}
}
this.ctx.putImageData(imgData1, 0, 0);
gray = this.readImage(this.canvas2);
rectContours = this.readContour(gray);
console.log(rectContours);
// get each number
const numbers = [];
rectContours.forEach(rect => {
this.drawContours2(rect);
numbers.push({
contour: rect,
matrix: gray.roi(rect)
});
});
// ##################### Traitement et prédiction par caractères ###########################
numbers.forEach((number, i) => {
this.clearCanvas2();
const rect = number.contour;
const matrix = number.matrix;
const width = rect.width;
const height = rect.height;
cv.cvtColor(matrix, matrix, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(matrix.cols, matrix.rows);
imageData1.data.set(new Uint8ClampedArray(matrix.data, matrix.cols, matrix.rows));
let rect2;
const padding = 10;
if (height > width) {
const border = Number((height - width) / 2);
this.ctx.putImageData(imageData1, padding + border, padding);
rect2 = new cv.Rect(0, 0, height + 2 * padding, height + 2 * padding);
} else {
const border = Number((width - height) / 2);
this.ctx.putImageData(imageData1, padding, border + padding);
rect2 = new cv.Rect(0, 0, width + 2 * padding, width + 2 * padding);
}
gray = this.readImage(this.canvas2);
gray1 = gray.roi(rect2);
this.clearCanvas2();
const dsize = new cv.Size(20, 20);
cv.threshold(gray1, gray1, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
cv.resize(gray1, gray1, dsize, 0, 0, cv.INTER_AREA);
cv.cvtColor(gray1, gray1, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(gray1.cols, gray1.rows);
imageData1.data.set(new Uint8ClampedArray(gray1.data, gray1.cols, gray1.rows));
// draw image on main canvas for debug
// this.context.putImageData(imageData1, i * 200, i * 200);
// Convert the canvas pixels to
let img = tf.browser
.fromPixels(imageData1)
// .resizeNearestNeighbor([20, 20])
.mean(2)
.expandDims(2)
.expandDims()
.toFloat();
// this.ctx.putImageData(img, i*200, i*200);
img = img.div(255.0);
// Make and format the predications
const output = this.model.predict(img) as any;
console.log(output);
const results = Array.from(output.dataSync());
let max = results[0];
let maxIndex = 0;
console.log(results);
for (let j = 1; j < results.length; j++) {
if (results[j] > max) {
maxIndex = j;
max = results[j];
}
}
if (maxIndex === 10) result = "1";
else if (maxIndex === 11) result = "2";
else if (maxIndex === 12) result = "3";
else if (maxIndex === 13) result = "4";
else if (maxIndex === 14) result = "5";
else if (maxIndex === 15) result = "6";
else if (maxIndex === 16) result = "7";
else if (maxIndex === 17) result = "9";
else if (maxIndex === 18) result = "+";
else if (maxIndex === 19) result = "-";
else if (maxIndex === 20) result = "x";
else if (maxIndex === 21) result = "/";
else if (maxIndex === 22) result = "=";
else result = maxIndex.toString();
console.log("max pred", max);
console.log("valeur pred", maxIndex);
resultsPrediction.push(result);
maxArray.push(max);
});
// gray1.delete();
const confidence = Math.min.apply(Math, maxArray);
const resultArray = new Array();
resultArray.push(resultsPrediction.toString().split(",").join(""));
resultArray.push(confidence as string);
console.error(resultArray);
subscriber.next(resultArray);
subscriber.complete();
});
}
drawContours(rect: any) {
// Stroked triangle
this.context.lineWidth = 3;
this.context.strokeStyle = "#11620d";
this.context.beginPath();
this.context.moveTo(rect.x - 5, rect.y - 5);
this.context.lineTo(rect.x + rect.width + 5, rect.y - 5);
this.context.lineTo(rect.x + rect.width + 5, rect.y + rect.height + 5);
this.context.lineTo(rect.x - 5, rect.y + rect.height + 5);
this.context.closePath();
this.context.stroke();
this.context.lineWidth = 20;
this.context.strokeStyle = "black";
}
drawContours2(rect: any) {
// Stroked triangle
this.ctx.lineWidth = 3;
this.ctx.strokeStyle = "#11620d";
this.ctx.beginPath();
this.ctx.moveTo(rect.x - 5, rect.y - 5);
this.ctx.lineTo(rect.x + rect.width + 5, rect.y - 5);
this.ctx.lineTo(rect.x + rect.width + 5, rect.y + rect.height + 5);
this.ctx.lineTo(rect.x - 5, rect.y + rect.height + 5);
this.ctx.closePath();
this.ctx.stroke();
this.ctx.lineWidth = 20;
this.ctx.strokeStyle = "black";
}
private createUserEvents() {
const canvas = this.canvas;
canvas.removeEventListener("mousedown", this.pressEventHandler);
canvas.removeEventListener("mousemove", this.dragEventHandler);
canvas.removeEventListener("mouseup", this.releaseEventHandler);
canvas.removeEventListener("mouseout", this.cancelEventHandler);
canvas.removeEventListener("touchstart", this.pressEventHandler);
canvas.removeEventListener("touchmove", this.dragEventHandler);
canvas.removeEventListener("touchend", this.releaseEventHandler);
canvas.removeEventListener("touchcancel", this.cancelEventHandler);
canvas.addEventListener("mousedown", this.pressEventHandler);
canvas.addEventListener("mousemove", this.dragEventHandler);
canvas.addEventListener("mouseup", this.releaseEventHandler);
canvas.addEventListener("mouseout", this.cancelEventHandler);
canvas.addEventListener("touchstart", this.pressEventHandler);
canvas.addEventListener("touchmove", this.dragEventHandler);
canvas.addEventListener("touchend", this.releaseEventHandler);
canvas.addEventListener("touchcancel", this.cancelEventHandler);
document.getElementById("clear").removeEventListener("click", this.clearEventHandler);
document.getElementById("clear").addEventListener("click", this.clearEventHandler);
}
private redraw() {
const clickX = this.clickX;
const context = this.context;
const clickDrag = this.clickDrag;
const clickY = this.clickY;
for (let i = 0; i < clickX.length; ++i) {
context.beginPath();
if (clickDrag[i] && i) {
context.moveTo(clickX[i - 1], clickY[i - 1]);
} else {
context.moveTo(clickX[i] - 1, clickY[i]);
}
context.lineTo(clickX[i], clickY[i]);
context.stroke();
}
context.closePath();
}
private addClick(x: number, y: number, dragging: boolean) {
this.clickX.push(x);
this.clickY.push(y);
this.clickDrag.push(dragging);
}
public clearCanvas() {
this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.clickX = [];
this.clickY = [];
this.clickDrag = [];
}
public clearCanvas2() {
this.ctx.clearRect(0, 0, this.canvas2.width, this.canvas2.height);
}
private clearEventHandler = () => {
this.clearCanvas();
};
private releaseEventHandler = () => {
this.paint = false;
this.redraw();
};
private cancelEventHandler = () => {
this.paint = false;
};
private pressEventHandler = (e: MouseEvent | TouchEvent) => {
let mouseX = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageX : (e as MouseEvent).pageX;
let mouseY = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageY : (e as MouseEvent).pageY;
mouseX -= this.canvas.offsetLeft;
mouseY -= this.canvas.offsetTop;
this.paint = true;
this.addClick(mouseX, mouseY, false);
this.redraw();
};
private dragEventHandler = (e: MouseEvent | TouchEvent) => {
let mouseX = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageX : (e as MouseEvent).pageX;
let mouseY = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageY : (e as MouseEvent).pageY;
mouseX -= this.canvas.offsetLeft;
mouseY -= this.canvas.offsetTop;
if (this.paint) {
this.addClick(mouseX, mouseY, true);
this.redraw();
}
e.preventDefault();
};
public updateSize() {
this.clearCanvas();
this.setCanvasSize();
this.configureContext();
}
public setCanvasSize() {
this.canvas.width = Math.floor((window.innerWidth * 80) / 100);
this.canvas.height = Math.floor(window.innerHeight - 90 - 50);
this.canvas2.width = Math.floor((window.innerWidth * 80) / 100);
this.canvas2.height = Math.floor(window.innerHeight - 90 - 50);
}
public configureContext() {
this.context = this.canvas.getContext("2d");
this.context.lineCap = "round";
this.context.lineJoin = "round";
this.context.strokeStyle = "black";
this.context.lineWidth = 20;
// this.ctx = this.canvas2.getContext("2d");
// this.ctx.lineCap = "round";
// this.ctx.lineJoin = "round";
// this.ctx.strokeStyle = "black";
// this.ctx.lineWidth = 20;
}
async playWithVideo(video){
// this.faceDetection();
this.ctx.drawImage(video, 0, 0, 640, 480);
const src = cv.imread(this.canvas2);
cv.cvtColor(src, src, cv.COLOR_RGBA2GRAY, 0);
cv.threshold(src, src, 120, 200, cv.THRESH_BINARY);
// const rectContours = this.readContour(src);
// rectContours.forEach(rect => {
// if(rect.width > 10 || rect.height > 10){
// this.drawContours(rect);
// console.error(rect);
// }
// })
let dst = cv.Mat.zeros(src.cols, src.rows, cv.CV_8UC3);
let contours = new cv.MatVector();
let hierarchy = new cv.Mat();
// You can try more different parameters
cv.findContours(src, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE);
console.error(contours.size());
// cv.findContours(src, contours, hierarchy, cv.RETR_CCOMP, cv.CHAIN_APPROX_SIMPLE);
for (let i = 0; i < contours.size(); ++i) {
let color = new cv.Scalar(Math.round(Math.random() * 255), Math.round(Math.random() * 255),
Math.round(Math.random() * 255));
cv.drawContours(dst, contours, i, color, 1, cv.LINE_8, hierarchy, 100);
}
cv.imshow('canvas', dst);
await AppUtils.timeOut(250);
this.playWithVideo(video);
}
faceDetection(){
const video = document.getElementById("live") as any;
const src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
const dst = new cv.Mat(video.height, video.width, cv.CV_8UC4);
const gray = new cv.Mat();
const cap = new cv.VideoCapture(video);
const faces = new cv.RectVector();
const classifier = new cv.CascadeClassifier();
// load pre-trained classifiers
classifier.load("/assets/tfjs/haarcascade_frontalface_default.xml");
const FPS = 30;
function processVideo() {
try {
const begin = Date.now();
// start processing.
cap.read(src);
src.copyTo(dst);
cv.cvtColor(dst, gray, cv.COLOR_RGBA2GRAY, 0);
// detect faces.
classifier.detectMultiScale(gray, faces, 1.1, 3, 0);
// draw faces.
for (let i = 0; i < faces.size(); ++i) {
const face = faces.get(i);
const point1 = new cv.Point(face.x, face.y);
const point2 = new cv.Point(face.x + face.width, face.y + face.height);
cv.rectangle(dst, point1, point2, [255, 0, 0, 255]);
}
cv.imshow(this.canvas, dst);
// schedule the next one.
const delay = 1000/FPS - (Date.now() - begin);
setTimeout(processVideo, 2000);
} catch (err) {
console.error(err);
}
};
// schedule the first one.
setTimeout(processVideo, 0);
}
}