import { HttpClient } from "@angular/common/http";
import { GlobalService } from "../services/global.service";
import { environment } from "../../environments/environment";
import { CabriDataService } from "src/app/services/cabri-data.service";
import { Platform } from "@ionic/angular";
import * as tf from "@tensorflow/tfjs-core";
import { Observable } from "rxjs";
import { AppUtils } from "../app-utils";
import { NetworkService } from "../services/network.service";
declare var window: { innerWidth: any; innerHeight: any };
declare var cv;
export class DrawingProd {
canvas2: HTMLCanvasElement;
private context: CanvasRenderingContext2D;
ctx: CanvasRenderingContext2D;
private paint: boolean;
private clickX: number[] = [];
private clickY: number[] = [];
private clickDrag: boolean[] = [];
img: HTMLImageElement;
public environment: { production: boolean; activityVersion: number; kidaia: boolean };
predictions: any;
public onlineRecognitionPromise: Observable<any> = null;
public onlineRecognitionResult: Object;
constructor(
private http: HttpClient,
public global: GlobalService,
public cabri: CabriDataService,
private platform: Platform,
private activity: string,
public networkService: NetworkService,
private canvas: HTMLCanvasElement
) {
this.environment = environment;
this.canvas2 = document.createElement("canvas") as HTMLCanvasElement;
// (window as any).document.querySelector("body").append(this.canvas2);
this.platform.ready().then(() => {
this.setCanvasSize(activity);
this.configureContext();
this.clearCanvas();
this.redraw();
this.createUserEvents();
});
}
textRecognition(currentNumber, training) {
const imgData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height);
// change non-opaque pixels to white
const data = imgData.data;
for (let i = 0; i < data.length; i += 4) {
if (data[i + 3] < 255) {
data[i] = 255;
data[i + 1] = 255;
data[i + 2] = 255;
data[i + 3] = 255;
}
}
this.context.putImageData(imgData, 0, 0);
const imgDataURL = this.canvas.toDataURL("image/jpeg");
const formData = new FormData();
formData.append("img", imgDataURL.replace(/^data:image\/(png|jpeg);base64,/, ""));
formData.append("currentNumber", currentNumber);
formData.append("training", training);
return this.http.post(
"https://mathia.education/wp-admin/admin-ajax.php?action=app_mathia_get_digit_recognition" +
(this.environment.production ? "" : "2"),
formData
);
}
tfjsTextRecognition(): Observable<string[]> {
return new Observable<string[]>(subscriber => {
this.global.loadOcrModel(this.networkService).subscribe(model => {
if (model) {
const imgData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height);
const resultsPrediction = new Array();
const maxArray = new Array();
let result = "";
//
// change non-opaque pixels to white
//
const data = imgData.data;
for (let i = 0; i < data.length; i += 4) {
if (data[i + 3] < 255) {
data[i] = 255;
data[i + 1] = 255;
data[i + 2] = 255;
data[i + 3] = 255;
}
}
this.clearCanvas2();
this.ctx.putImageData(imgData, 0, 0);
//
// lire l'image et extraire les contours
//
const imgDataURL = this.canvas2.toDataURL("image/jpeg");
if(this.networkService.isConnected){
// send image to dataset
const formData = new FormData();
formData.append("img", imgDataURL.replace(/^data:image\/(png|jpeg);base64,/, ""));
formData.append("builddataset", "true");
this.onlineRecognitionPromise = this.http.post(
"https://mathia.education/wp-admin/admin-ajax.php?action=app_mathia_get_digit_recognition" +
(this.environment.production ? "" : "2"),
formData
);
this.onlineRecognitionPromise.subscribe((e)=>{
this.onlineRecognitionResult = e;
console.log("image saved", e);
});
}
let gray = new cv.Mat();
let gray1 = new cv.Mat();
gray = this.readImage(this.canvas2);
let rectContours = this.readContour(gray);
this.clearCanvas2();
// ################### Redessiner l'image pour le centrer ###########################
cv.threshold(gray, gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
let diff = 0;
let imageData1 = imgData;
rectContours.forEach(rect => {
console.error(rect);
const border = 0;
if (rect.height > 50) {
this.drawContours(rect);
}
let width = rect.width;
let height = rect.height;
if(rect.x + width >= gray.cols){
width = gray.cols - rect.x - 1;
}
if(rect.y + height >= gray.rows){
height = gray.rows - rect.y - 1;
}
rect = new cv.Rect(rect.x, rect.y, width, height);
gray1 = gray.roi(rect);
// console.log(imgData.data);
// ++++++ Dimensions de l'image à redessiner ++++++++++
let newWidth = width;
let newheight = height;
// ++++++++ Cas où l'image prend tout le canvas ++++++++++
// if (width > gray.cols / 3) newWidth = gray.cols / 3;
// if (height > gray.rows / 3) newheight = gray.rows / 3;
const dsize = new cv.Size(newWidth, newheight);
// ++++++++++ Cas où la hauteur est 2 fois plus grand que la largeur +++++++++
// if (newheight > (newWidth * 2))
// dsize = new cv.Size(newWidth, newWidth * 2);
cv.resize(gray1, gray1, dsize, 0, 0, cv.INTER_AREA);
cv.cvtColor(gray1, gray1, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(gray1.cols, gray1.rows);
imageData1.data.set(new Uint8ClampedArray(gray1.data, gray1.cols, gray1.rows));
this.ctx.putImageData(imageData1, 1 + diff, 1);
diff = diff + newWidth + 100;
});
// ################## Recupérer l'image centrée ########################
const imgData1 = this.ctx.getImageData(0, 0, this.canvas2.width, this.canvas2.height);
const data1 = imgData1.data;
// change non-opaque pixels to white
for (let i = 0; i < data1.length; i += 4) {
if (data1[i + 3] < 255) {
data1[i] = 255;
data1[i + 1] = 255;
data1[i + 2] = 255;
data1[i + 3] = 255;
}
}
this.ctx.putImageData(imgData1, 0, 0);
gray = this.readImage(this.canvas2);
rectContours = this.readContour(gray);
console.log(rectContours);
// get each number
const numbers = [];
rectContours.forEach(rect => {
numbers.push({
contour: rect,
matrix: gray.roi(rect)
});
});
// ##################### Traitement et prédiction par caractères ###########################
numbers.forEach(async (number, i) => {
this.clearCanvas2(true);
const rect = number.contour;
const matrix = number.matrix;
let width = rect.width;
let height = rect.height;
// don't predict too small object
if (height > 50) {
cv.cvtColor(matrix, matrix, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(matrix.cols, matrix.rows);
imageData1.data.set(new Uint8ClampedArray(matrix.data, matrix.cols, matrix.rows));
let rect2;
const padding = 10;
if (height > width) {
const border = Number((height - width) / 2);
this.ctx.putImageData(imageData1, padding + border, padding);
rect2 = new cv.Rect(0, 0, height + 2 * padding, height + 2 * padding);
} else {
const border = Number((width - height) / 2);
this.ctx.putImageData(imageData1, padding, border + padding);
rect2 = new cv.Rect(0, 0, width + 2 * padding, width + 2 * padding);
}
gray = this.readImage(this.canvas2);
if(rect2.x + width >= gray.cols){
rect2.width = gray.cols - rect2.x - 1;
}
if(rect2.y + height >= gray.rows){
rect2.height = gray.rows - rect2.y - 1;
}
gray1 = gray.roi(rect2);
this.clearCanvas2(true);
const dsize = new cv.Size(20, 20);
cv.threshold(gray1, gray1, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
cv.resize(gray1, gray1, dsize, 0, 0, cv.INTER_AREA);
cv.cvtColor(gray1, gray1, cv.COLOR_GRAY2RGBA);
imageData1 = this.ctx.createImageData(gray1.cols, gray1.rows);
imageData1.data.set(new Uint8ClampedArray(gray1.data, gray1.cols, gray1.rows));
// draw image on main canvas for debug
// this.context.putImageData(imageData1, i * 200, i * 200);
// Convert the canvas pixels to
let img = tf.browser
.fromPixels(imageData1)
// .resizeNearestNeighbor([20, 20])
.mean(2)
.expandDims(2)
.expandDims()
.toFloat();
// this.ctx.putImageData(img, i*200, i*200);
img = img.div(255.0);
// Make and format the predications
let output = null;
try {
output = model.predict(img) as any;
} catch {
if (tf.getBackend() !== "cpu") {
tf.setBackend("cpu");
await tf.ready();
output = model.predict(img) as any;
} else {
//TODO here we can't predict so we need to remove the drawing method
subscriber.error("Can't predicte");
}
}
// console.log(output);
const results = Array.from(output.dataSync());
let max = results[0];
let maxIndex = 0;
// console.log(results);
for (let j = 1; j < results.length; j++) {
if (results[j] > max) {
maxIndex = j;
max = results[j];
}
}
if (maxIndex === 10) result = "1";
else if (maxIndex === 11) result = "2";
else if (maxIndex === 12) result = "3";
else if (maxIndex === 13) result = "4";
else if (maxIndex === 14) result = "5";
else if (maxIndex === 15) result = "6";
else if (maxIndex === 16) result = "7";
else if (maxIndex === 17) result = "9";
else if (maxIndex === 18) result = "+";
else if (maxIndex === 19) result = "-";
else if (maxIndex === 20) result = "x";
else if (maxIndex === 21) result = "/";
else if (maxIndex === 22) result = "=";
else result = maxIndex.toString();
// console.log("max pred", max);
// console.log("valeur pred", maxIndex);
resultsPrediction.push(result);
maxArray.push(max);
}
});
// gray1.delete();
const confidence = Math.min.apply(Math, maxArray);
const resultArray = new Array();
resultArray.push(resultsPrediction.toString().split(",").join(""));
resultArray.push(confidence as string);
console.log("OCR results", resultArray, maxArray);
AppUtils.timeOut(500).then(() => {
subscriber.next(resultArray);
subscriber.complete();
this.clearCanvas();
});
} else {
//TODO here model can't be loaded so we need to remove the drawing method
}
});
});
}
/**
* OpenCV find contours
*/
readContour(image) {
const contours = new cv.MatVector();
const hierarchy = new cv.Mat();
cv.findContours(image, contours, hierarchy, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_NONE);
console.log("contours size ", contours.size());
const rectContours = new Array();
for (let i = 0; i < contours.size(); i++) {
// You can try more different parameters
const rect = cv.boundingRect(contours.get(i));
rectContours.push(rect);
}
contours.delete();
hierarchy.delete();
rectContours.sort(this.compare);
return rectContours;
}
// pour comparer les coordonnées x des rectangles
compare(a, b) {
if (a.x > b.x) return 1;
if (b.x > a.x) return -1;
return 0;
}
readImage(canvas) {
const src = cv.imread(canvas);
const gray = new cv.Mat();
cv.cvtColor(src, gray, cv.COLOR_RGBA2GRAY);
cv.threshold(gray, gray, 0, 255, cv.THRESH_BINARY_INV + cv.THRESH_OTSU);
src.delete();
return gray;
}
public clearCanvas2(transparent = false) {
if(transparent){
this.ctx.clearRect(0, 0, this.canvas2.width, this.canvas2.height);
} else {
this.ctx.fillRect(0, 0, this.canvas2.width, this.canvas2.height);
}
}
drawContours(rect: any) {
// console.error("drawing rect", rect);
// Stroked triangle
this.context.lineWidth = 3;
this.context.strokeStyle = "#00EDC8";
this.context.beginPath();
this.context.moveTo(rect.x - 5, rect.y - 5);
this.context.lineTo(rect.x + rect.width + 5, rect.y - 5);
this.context.lineTo(rect.x + rect.width + 5, rect.y + rect.height + 5);
this.context.lineTo(rect.x - 5, rect.y + rect.height + 5);
this.context.closePath();
this.context.stroke();
this.context.lineWidth = this.global.isMobile ? 10 : 20;
this.context.strokeStyle = "black";
}
private createUserEvents() {
const canvas = this.canvas;
canvas.removeEventListener("mousedown", this.pressEventHandler);
canvas.removeEventListener("mousemove", this.dragEventHandler);
canvas.removeEventListener("mouseup", this.releaseEventHandler);
canvas.removeEventListener("mouseout", this.cancelEventHandler);
canvas.removeEventListener("touchstart", this.pressEventHandler);
canvas.removeEventListener("touchmove", this.dragEventHandler);
canvas.removeEventListener("touchend", this.releaseEventHandler);
canvas.removeEventListener("touchcancel", this.cancelEventHandler);
canvas.addEventListener("mousedown", this.pressEventHandler);
canvas.addEventListener("mousemove", this.dragEventHandler);
canvas.addEventListener("mouseup", this.releaseEventHandler);
canvas.addEventListener("mouseout", this.cancelEventHandler);
canvas.addEventListener("touchstart", this.pressEventHandler);
canvas.addEventListener("touchmove", this.dragEventHandler);
canvas.addEventListener("touchend", this.releaseEventHandler);
canvas.addEventListener("touchcancel", this.cancelEventHandler);
document.getElementById("clear").removeEventListener("click", this.clearEventHandler);
document.getElementById("clear").addEventListener("click", this.clearEventHandler);
}
private redraw() {
const clickX = this.clickX;
const context = this.context;
const clickDrag = this.clickDrag;
const clickY = this.clickY;
for (let i = 0; i < clickX.length; ++i) {
context.beginPath();
if (clickDrag[i] && i) {
context.moveTo(clickX[i - 1], clickY[i - 1]);
} else {
context.moveTo(clickX[i] - 1, clickY[i]);
}
context.lineTo(clickX[i], clickY[i]);
context.stroke();
}
context.closePath();
}
private addClick(x: number, y: number, dragging: boolean) {
this.clickX.push(x);
this.clickY.push(y);
this.clickDrag.push(dragging);
}
public clearCanvas() {
this.context.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.clickX = [];
this.clickY = [];
this.clickDrag = [];
}
private clearEventHandler = () => {
this.clearCanvas();
};
private releaseEventHandler = () => {
this.paint = false;
this.redraw();
};
private cancelEventHandler = () => {
this.paint = false;
};
private pressEventHandler = (e: MouseEvent | TouchEvent) => {
let mouseX = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageX : (e as MouseEvent).pageX;
let mouseY = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageY : (e as MouseEvent).pageY;
mouseX -= this.canvas.offsetLeft;
mouseY -= this.canvas.offsetTop;
this.paint = true;
this.addClick(mouseX, mouseY, false);
this.redraw();
};
private dragEventHandler = (e: MouseEvent | TouchEvent) => {
let mouseX = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageX : (e as MouseEvent).pageX;
let mouseY = (e as TouchEvent).changedTouches ? (e as TouchEvent).changedTouches[0].pageY : (e as MouseEvent).pageY;
mouseX -= this.canvas.offsetLeft;
mouseY -= this.canvas.offsetTop;
if (this.paint) {
this.addClick(mouseX, mouseY, true);
this.redraw();
}
e.preventDefault();
};
public updateSize(activity: string) {
this.clearCanvas();
this.setCanvasSize(activity);
this.configureContext();
}
public setCanvasSize(activity: string = "jeudufuret") {
if (document && document.getElementById("canvasProd") && this.canvas) {
// this.canvas.width = Math.floor((window.innerWidth * 78) / 100);
this.canvas.width = Math.floor((window.innerWidth * 98) / 100);
let marginBottom = this.global.toolbarHeight + this.global.progressbarHeight + 4;
let canvasHeightBeforeMargin: number;
if (activity === "calculmental" && this.cabri.mirrorMode !== true) {
if (this.cabri.collectionModeInCM) {
canvasHeightBeforeMargin = Math.floor((window.innerHeight * 84) / 100);
marginBottom = this.global.toolbarHeight + 8 + 14;
} else {
marginBottom = this.global.toolbarHeight + this.global.progressbarHeight;
if (this.global.isMobile) {
canvasHeightBeforeMargin = Math.floor((window.innerHeight * 82.5) / 100);
} else {
canvasHeightBeforeMargin = Math.floor((window.innerHeight * 62) / 100);
}
}
} else if (activity === "calculmental" && this.cabri.mirrorMode) {
marginBottom = this.global.isMobile ? this.global.toolbarHeight + 12 : this.global.toolbarHeight + 4;
canvasHeightBeforeMargin = Math.floor((window.innerHeight * 82) / 100);
} else if (activity === "jeudufuret" || activity === "jeudekim" || activity === "remote" || activity === "galaxy") {
canvasHeightBeforeMargin = Math.floor((window.innerHeight * 82) / 100);
}
this.canvas.height = canvasHeightBeforeMargin - marginBottom;
if (this.canvas2) {
this.canvas2.width = this.canvas.width * 2;
this.canvas2.height = this.canvas.height + 100;
}
}
}
public configureContext() {
if (this.canvas) {
this.context = this.canvas.getContext("2d");
this.context.lineCap = "round";
this.context.lineJoin = "round";
this.context.strokeStyle = "black";
this.context.lineWidth = this.global.isMobile ? 10 : 20;
}
if (this.canvas2) {
this.ctx = this.canvas2.getContext("2d");
this.ctx.fillStyle = "#FFFFFF";
}
}
}