/// <reference path="../../../web1/ui1.lib/innovaphone.ui1.lib.js" />

var innovaphone = innovaphone || {};
innovaphone.VirtualBackground = innovaphone.VirtualBackground || function (pathToConfig) {
    console.log("VirtualBackground: " + "virtualbackground library has started");
    let that = this;
    that.config = pathToConfig;

    // load modules and js asynchronously
    let vb = null;
    let glfx = false;
    let vision = null;
    let segmenter = {
        simd: {},
        nosimd: {},
    }
    let baseUrl = "";

    // check if modules are loaded
    function isModuleLoaded() {
        return vision != null && glfx;
    }

    // background modes (enum)
    const bmode = {
        STREAM: 0,                              // uses current background as background
        COLOR: 1,                               // uses a defined color value as background
        IMAGE: 2                                // uses a defined image as background
    };

    // calculation modes (enum)
    const cmode = {
        CPU: "CPU",                            // delegate mask calculation to cpu
        GPU: "GPU"                             // delegate mask calculation to gpu (recommended)
    }

    // quality modes (enum)
    const qmode = {
        MAX: 0,                                 // using input quality for mask creation (affects only mask, not recommended)
        CUSTOM: 1,                              // using custom quality mode defined in config
        HIGH: 2,                                // using high quality (256 x 256) for mask creation (affects only mask, recommended)
        MEDIUM: 3,                              // using medium quality (128 x 128) for mask creation (affects only mask)
        LOW: 4,                                 // using low quality (64 x 64) for mask creation (affects only mask)
        AUTO: 5                                 // chooses automaticallly the right mask creation depending on the device performance
    }

    // drawing modes (enum)
    const dmode = {
        FIT: 0,                                 // draw image in fit mode (required for processing frames)
        CUT: 1,                                 // draw image in cut mode (required for processing frames)
        SCALE: 2,                               // draw image in scale mode (required for processing frames)
    }

    const MASK_BLUR_RADIUS = 4;
    const HIGHEST_QMODE = qmode.HIGH;
    const LOWEST_QMODE = qmode.MEDIUM;

    // state machine
    const INITIALIZE_MODULES = 0
    const LOAD_MODULES = 1;
    const INITIALIZE_SEGMENTER = 2;
    const LOAD_SEGMENTER = 3;
    const SEGMENTATION = 4;
    const INITIALIZATION_FAILED = 5;
    const SEGMENTATION_NOT_REQUIRED = 6;
    const SEGMENTATION_NOT_POSSIBLE = 7;
    const TERMINATED = 8;

    // worker implementation
    console.log("VirtualBackground: " + "creating worker object url");
    const blob = new Blob([" \
        function onTimeout() { \
            postMessage(\"timeout\"); \
        } \
        onmessage = function (e) { \
            setTimeout(onTimeout, Number(e.data[0])); \
        } \
        "],
        { type: "text/javascript" },
    );
    const workerURL = URL.createObjectURL(blob);
    console.log("VirtualBackground: " + "worker object url created");

    // optional method to setting configuration path in a later step 
    // (using virtual background with pathToConfig = null will result in an error during segmentation)
    that.setPathToConfig = function (pathToConfig) {
        that.config = pathToConfig;
    }

    // get current configuration path
    that.getPathToConfig = function () {
        return that.config;
    }

    // creates a table containing configurations for processing the stream and stream background in the process step
    // also creates member functions in the table to securely modify and access configurations
    that.prepare = function (stream) {
        let obj = {
            stream: stream,                     // input stream
            capturedStream: null,               // output stream
            canvas: null,                       // canvas element used for the output
            bCanvas: null,                      // canvas element to buffer drawing operations offscreen
            errorCallbacks: {},                 // dictionary where error callbacks for an object are stored 
            glfx: null,                         // dictionary where all background canvas elements are stored
            video: null,                        // video element used for the input
            state: INITIALIZE_MODULES,          // state machine for the segmentation task
            simd: null,                         // simd | nosimg, required to determine the loaded segmenter
            qCanvas: null,                      // quality mode canvas
            autoAdjustQuality: null,            // stores the timeout identifer for autoqmode
            requestId: null,                    // id for determining active timeout loop created by the process function
            worker: null,                       // worker that prevents setTimeout to be limited to 1 fps if tab is inactive
            config: {                           // background configuration
                bmode: bmode.STREAM,                // background mode
                cmode: cmode.GPU,                   // calc mode
                qmode: qmode.HIGH,                  // quality mode
                dmode: dmode.CUT,                   // draw mode
                autoqmode: true,                    // auto qmode active
                smoothing: 2,                       // enable smoothing
                fps: 60,                            // fps limit (used with timeout functionality (replaces requestAnimationFrame to prevent stream stop while changing focus))
                color: "rgb(255, 255, 255, 255)",   // background color, required for "COLOR" bmode
                image: null,                        // background image, required for "IMAGE" bmode
                dmodeImage: dmode.CUT,              // draw mode for background images
                blur: 0,                            // blur effect value (value is interpreted differently depending on the render mode)
                grayscale: false,                   // grayscale effect
                sepia: false                        // sepia effect
            },
        };

        // stops previous streams, process and return a new stream
        obj.process = function (fps = 60, width, height) {
            if (width && isNaN(width) || width && Number(width) <= 0) {
                console.error("VirtualBackgroundObject.process failed: " + "invalid width " + width);
                throw Error("invalid width");
            }
            if (height && isNaN(height) || height && Number(height) <= 0) {
                console.error("VirtualBackgroundObject.process failed: " + "invalid height" + height);
                throw Error("invalid height");
            }
            this.setFPS(fps);
            this.stop();
            if (!this.video) {
                this.video = document.createElement("video");
                this.video.autoplay = true;
                this.video.playsInline = true;
            }
            this.state = INITIALIZE_MODULES;
            this.video.srcObject = this.stream;
            this.canvas = this.canvas || document.createElement("canvas");
            this.bCanvas = this.bCanvas || document.createElement("canvas");
            this.video.addEventListener("play", () => {
                this.canvas.width = Number(width) || this.video.videoWidth;
                this.canvas.height = Number(height) || this.video.videoHeight;
                this.bCanvas.width = this.video.videoWidth;
                this.bCanvas.height = this.video.videoHeight;
                this.qCanvas = this.qCanvas || createCanvas(this.video.videoWidth, this.video.videoHeight);
                processVideo(this);
            });
            if (window.Worker) {
                if (this.worker) {
                    this.worker.terminate();
                    delete this.worker;
                    this.worker = null;
                }
                this.worker = new Worker(workerURL);
            }
            console.log("VirtualBackgroundObject.process success: " + JSON.stringify({ width: width, height: height, fps: fps, worker: this.worker != null }));
            if (this.capturedStream) {
                this.capturedStream.getTracks().forEach(function (track) {
                    track.stop();
                });
            }
            this.capturedStream = this.canvas.captureStream();
            return this.capturedStream;
        }

        // stops previous streams if there is any
        obj.stop = function () {
            if (this.requestId) {
                clearTimeout(this.requestId);
            }
            if (this.autoAdjustQuality) {
                clearTimeout(this.autoAdjustQuality);
            }
            this.requestId = null;
            this.autoAdjustQuality = null;
            if (this.worker) {
                this.worker.terminate();
                delete this.worker;
                this.worker = null;
            }
            this.state = TERMINATED;
            console.log("VirtualBackgroundObject.stop success: " + "obj worker and timeouts stopped");
            return this;
        }

        // stops input stream
        obj.stopStream = function () {
            this.stream.getTracks().forEach(function (track) {
                track.stop();
            });
        }

        // replace stream while unchanging configuration
        obj.replaceStream = function (stream) {
            this.stream = stream;
            console.log("VirtualBackgroundObject.replaceStream success: " + "stream replaced");
            return this;
        }

        // set canvas width and height during process (no effect before)
        obj.setDimensions = function (width, height) {
            if (width && (isNaN(width) || Number(width) <= 0)) {
                console.error("VirtualBackgroundObject.setDimensions failed: " + "invalid width");
                throw Error("invalid width");
            }
            if (height && (isNaN(height) || Number(height) <= 0)) {
                console.error("VirtualBackgroundObject.setDimensions failed: " + "invalid height");
                throw Error("invalid height");
            }
            if (!this.canvas || !this.video) {
                console.warn("VirtualBackgroundObject.setDimensions failed: " + "set dimensions is only possible during a stream. it will be overwritten by process");
                return;
            }
            width = width || this.video.videoWidth;
            height = height || this.video.videoHeight;
            this.canvas.width = Number(width);
            this.canvas.height = Number(height);
            console.log("VirtualBackgroundObject.setDimensions success: " + JSON.stringify({ width: width, height: height }));
            return this;
        }

        // resets the internal state of the virtualbackground obj to init modules
        obj.reload = function () {
            this.config.state = INITIALIZE_MODULES;
            return this;
        }

        // set blur filter
        obj.setBlur = function (value = 3) {
            if (isNaN(value) || value < 0) {
                console.error("VirtualBackgroundObject.setBlur failed: " + "invalid value");
                throw Error("invalid value");
            }
            this.config.blur = Number(value);
            console.log("VirtualBackgroundObject.setBlur success: " + JSON.stringify({ blur: value }));
            return this;
        };

        // apply grayscale and negate sepia filter
        obj.applyGrayscale = function () {
            this.config.grayscale = true;
            this.removeSepia();
            console.log("VirtualBackgroundObject.applyGrayscale success: " + "grayscale applied");
            return this;
        }

        // negate grayscale filter
        obj.removeGrayscale = function () {
            this.config.grayscale = false;
            console.log("VirtualBackgroundObject.removeGrayscale success: " + "grayscale removed");
            return this;
        }

        // apply sepia and negate grayscale filter
        obj.applySepia = function () {
            this.config.sepia = true;
            this.removeGrayscale();
            console.log("VirtualBackgroundObject.applySepia success: " + "sepia applied");
            return this;
        }

        // negate sepia filter
        obj.removeSepia = function () {
            this.config.sepia = false;
            console.log("VirtualBackgroundObject.removeSepia success: " + "sepia removed");
            return this;
        }

        // reset all filter
        obj.resetFilter = function () {
            this.config.blur = 0;
            this.config.sepia = false;
            this.config.grayscale = false;
            console.log("VirtualBackgroundObject.resetFilter success: " + "filter resetted");
            return this;
        }

        // set stream background (default)
        obj.setStreamBackground = function () {
            this.config.bmode = bmode.STREAM;
            console.log("VirtualBackgroundObject.setStreamBackground success: " + "using stream background");
            return this;
        }

        // set color background
        obj.setColorBackground = function (r = 255, g = 255, b = 255) {
            if (isNaN(r) || isNaN(g) || isNaN(b) || r < 0 || g < 0 || b < 0 || r > 255 || g > 255 || b > 255) {
                console.error("VirtualBackgroundObject.setColorBackground failed: " + "invalid input " + JSON.stringify({ r: r, g: g, b: b }));
                throw Error("invalid input");
            }
            this.config.bmode = bmode.COLOR;
            this.config.color = "rgb(" + r + ", " + g + ", " + b + ", 255)";
            console.log("VirtualBackgroundObject.setColorBackground success: " + JSON.stringify({ r: r, g: g, b: b }));
            return this;
        }

        // set image background
        obj.setImageBackground = function (imagePath, mode = "CUT") {
            let that = this;
            let val = dmode[mode];
            if (!imagePath) {
                this.setColorBackground(0, 0, 0);
                return;
            }
            if (typeof imagePath != "string") {
                console.error("VirtualBackgroundObject.setImageBackground failed: " + "invalid input " + JSON.stringify({ imagePath: imagePath, mode: mode }));
                throw Error("invalid input");
            }
            if (isNaN(val)) {
                console.error("VirtualBackgroundObject.setImageBackground failed: " + "invalid mode " + JSON.stringify({ imagePath: imagePath, mode: mode }));
                throw Error("invalid input");
            }
            if (this.config.dmodeImage != val) {
                this.config.dmodeImage = val;
            }
            this.setColorBackground(0, 0, 0);           // set color background until it is clear that image is an actual image
            this.config.image = document.createElement("img");
            this.config.image.addEventListener("load", function (ev) {
                if (this.width && this.height && this.width != 0 && this.height != 0) {
                    that.config.imageAspectRatio = this.width / this.height;
                    that.config.bmode = bmode.IMAGE;
                    console.log("VirtualBackgroundObject.setImageBackground success: " + "image loaded");
                } else {
                    console.error("VirtualBackgroundObject.setImageBackground failed: " + "input is probably not an image");
                }
            });
            this.config.image.addEventListener("error", function (ev) {
                console.error("VirtualBackgroundObject.setImageBackground failed: " + ev);
            });
            this.config.image.src = imagePath;
            console.log("VirtualBackgroundObject.setImageBackground pending: " + imagePath);
            return this;
        }

        // set stream fps
        obj.setFPS = function (fps = 60) {
            if (isNaN(fps) || fps <= 0) {
                console.error("VirtualBackgroundObject.setFPS failed: " + "invalid input" + JSON.stringify({ fps: fps }));
                throw Error("invalid input");
            }
            this.config.fps = Number(fps);
            console.log("VirtualBackgroundObject.setFPS success: " + JSON.stringify({ fps: fps }));
            return this;
        }

        // enable smoothing
        obj.setSmoothing = function (val = 2) {
            this.config.smoothing = val;
            console.log("VirtualBackgroundObject.setSmoothing success: " + JSON.stringify({ smoothing: val }));
            return this;
        }

        // set error callback
        obj.setErrorCallback = function (identifier, callback) {
            if (!identifier) {          // handle missing identifier
                console.error("VirtualBackgroundObject.setErrorCallback failed: " + "invalid input for " + identifier);
                throw Error("invalid input");
            } else if (!callback) {     // handle deletion
                if (this.errorCallbacks[identifier]) {
                    delete this.errorCallbacks[identifier];
                }
                return this;
            }
            if (typeof callback !== "function") {       // handle missing callback if no deletion has happened
                console.error("VirtualBackgroundObject.setErrorCallback failed: " + "invalid input for " + identifier);
                throw Error("invalid input");
            }
            this.errorCallbacks[identifier] = callback; // adding callback
            console.log("VirtualBackgroundObject.setErrorCallback success: " + "callback added for " + identifier);
            return this;
        }

        // set calculation mode
        obj.setCalculationMode = function (mode = "GPU") {
            let val = cmode[mode];
            if (!val) {
                console.error("VirtualBackgroundObject.setCalculationMode failed: " + "invalid mode " + JSON.stringify({ mode: mode }));
                throw Error("invalid input");
            }
            if (this.config.cmode == val) {
                console.log("VirtualBackgroundObject.setCalculationMode success: " + JSON.stringify({ mode: mode }));
                return this;                    // no modification required
            }
            if (this.simd && isModuleLoaded()) {
                const wasmFileset = {
                    wasmLoaderPath: baseUrl + vb.config.wasm[this.simd].js,
                    wasmBinaryPath: baseUrl + vb.config.wasm[this.simd].file
                }
                vision.ImageSegmenter.createFromOptions(wasmFileset, {
                    baseOptions: {
                        modelAssetPath: baseUrl + this.config.model,
                        delegate: val
                    },
                    outputCategoryMask: false, // required for category mask usage
                    outputConfidenceMasks: true, // required for confidence mask usage
                    runningMode: "VIDEO"
                }).then(function (sgmtr) {
                    segmenter[this.simd][val] = sgmtr;
                    this.config.cmode = val;
                    console.log("VirtualBackgroundObject.setCalculationMode success: " + JSON.stringify({ mode: mode }));
                });
            } else {
                this.config.cmode = val;
                console.log("VirtualBackgroundObject.setCalculationMode success: " + JSON.stringify({ mode: mode }));
            }
            return this;
        }

        // set stream drawing mode
        obj.setDrawingMode = function (mode = "CUT") {
            let val = dmode[mode];
            if (isNaN(val)) {
                console.error("VirtualBackgroundObject.setDrawingMode failed: " + "invalid mode " + JSON.stringify({ mode: mode }));
                throw Error("invalid input");
            }
            if (this.config.dmode == val) {
                console.log("VirtualBackgroundObject.setDrawingMode success: " + JSON.stringify({ mode: mode }));
                return this;                    // no modification required
            }
            this.config.dmode = val;
            console.log("VirtualBackgroundObject.setDrawingMode success: " + JSON.stringify({ mode: mode }));
            return this;
        }

        // set quality mode
        obj.setQualityMode = function (mode = "HIGH") {
            let val = qmode[mode];
            if (isNaN(val) || val == qmode.CUSTOM) {
                console.error("VirtualBackgroundObject.setQualityMode failed: " + "invalid mode " + JSON.stringify({ mode: mode }));
                throw Error("invalid input");
            }
            this.config.autoqmode = val == qmode.AUTO;
            if (this.config.autoqmode || this.config.qmode == val) {
                console.log("VirtualBackgroundObject.setQualityMode success: " + JSON.stringify({ mode: mode }));
                return this;                    // no modification required
            }
            console.log("VirtualBackgroundObject.setQualityMode success: " + JSON.stringify({ mode: mode }));
            return this.setQualityModeInternal(val);
        }

        // set custom quality mode
        obj.setCustomQualityMode = function (width = 256, height = 256) {
            if (width < 0 || height < 0) {
                console.error("VirtualBackgroundObject.setCustomQualityMode failed: " + "invalid input " + JSON.stringify({ width: width, height: height }));
                throw Error("invalid input");
            }
            if (!this.qCanvas) {
                this.qCanvas = createCanvas(Number(width), Number(height));
            } else {
                if (this.qCanvas.width == Number(width) && this.qCanvas.height == Number(height) && this.config.qmode != qmode.MAX) {
                    return this;                    // no modification required
                }
                this.qCanvas.width = Number(width);
                this.qCanvas.height = Number(height);
            }
            this.config.qmode = qmode.CUSTOM;
            console.log("VirtualBackgroundObject.setCustomQualityMode success: " + JSON.stringify({ width: width, height: height }));
            return this;
        }

        // internal calling error callbacks
        obj.handleErrorInternal = function (msg) {
            Object.keys(this.errorCallbacks).forEach((key) => {
                console.log("VirtualBackgroundObject.handleError calling callback for: " + key);
                this.errorCallbacks[key](msg);
            })
        }

        // internal set quality mode function (for automatical adjustments)
        obj.setQualityModeInternal = function (val = qmode.HIGH) {
            let width, height;
            this.config.qmode = val;
            switch (val) {
                case qmode.HIGH:
                    width = 256;
                    height = 256;
                    break;
                case qmode.MEDIUM:
                    width = 128;
                    height = 128;
                    break;
                case qmode.LOW:
                    width = 64;
                    height = 64;
                    break;
                default:
                    if (this.video) {
                        width = this.video.videoWidth;
                        height = this.video.videoHeight;
                    } else {
                        return this;
                    }
                    break;
            }
            if (!this.qCanvas) {
                this.qCanvas = createCanvas(width, height);
            } else {
                if (this.qCanvas.width == width && this.qCanvas.height == height) {
                    return;
                }
                this.qCanvas.width = width;
                this.qCanvas.height = height;
            }
            console.log("VirtualBackgroundObject.setCustomQualityModeInternal success: " + JSON.stringify({ width: width, height: height }));
            return this;
        }

        console.log("VirtualBackground::prepare success: " + "virtualbackground object created");
        return obj;
    }

    // replaces ctx.drawImage using a 2d context
    function drawImage2d(output, input, mode = dmode.FIT, ctx) {
        if (!input) {
            drawColor2d(output, { r: 0, g: 0, b: 0 });
        }
        const outputWidth = output.width;
        const outputHeight = output.height;
        const inputWidth = input.videoWidth || input.width;
        const inputHeight = input.videoHeight || input.height;
        ctx = ctx || output.getContext("2d");
        if (outputWidth == 0 || outputHeight == 0) { return; }
        if (inputWidth == 0 || inputHeight == 0) { return; }
        switch (mode) {
            case dmode.CUT:
                {
                    const outputAspectRatio = outputWidth / outputHeight;
                    const inputAspectRatio = inputWidth / inputHeight;
                    if (outputAspectRatio == inputAspectRatio) {                            // input and output has the same aspect ratio
                        ctx.drawImage(input, 0, 0, outputWidth, outputHeight);
                    } else if (outputAspectRatio < inputAspectRatio) {                      // output has a smaller aspect ratio
                        const scale = outputHeight / inputHeight;
                        const w = inputWidth * scale;
                        const x = (outputWidth - w) / 2;
                        ctx.drawImage(input, x, 0, w, outputHeight);
                    } else if (outputAspectRatio > inputAspectRatio) {                      // output has a larger aspect ratio
                        const scale = outputWidth / inputWidth;
                        const h = inputHeight * scale;
                        const y = (outputHeight - h) / 2;
                        ctx.drawImage(input, 0, y, outputWidth, h);
                    }
                }
                break;
            case dmode.SCALE:
                {
                    const outputAspectRatio = outputWidth / outputHeight;
                    const inputAspectRatio = inputWidth / inputHeight;
                    ctx.fillStyle = "black";
                    ctx.fillRect(0, 0, outputWidth, outputHeight);
                    if (outputAspectRatio == inputAspectRatio) {                            // input and output has the same aspect ratio
                        ctx.drawImage(input, 0, 0, outputWidth, outputHeight);
                    } else if (outputAspectRatio < inputAspectRatio) {                      // output has a smaller aspect ratio
                        const scale = outputWidth / inputWidth;
                        const h = inputHeight * scale;
                        const y = (outputHeight - h) / 2;
                        ctx.drawImage(input, 0, y, outputWidth, h);
                    } else if (outputAspectRatio > inputAspectRatio) {                      // output has a larger aspect ratio
                        const scale = outputHeight / inputHeight;
                        const w = inputWidth * scale;
                        const x = (outputWidth - w) / 2;
                        ctx.drawImage(input, x, 0, w, outputHeight);
                    }
                }
                break;
            default:
                ctx.drawImage(input, 0, 0, outputWidth, outputHeight);
                break;
        }
    }

    function createCanvas(width, height) {
        const canvas = document.createElement("canvas");
        canvas.width = width;
        canvas.height = height;
        return canvas;
    }

    function resizeCanvas(canvas, width, height) {
        canvas.width = width;
        canvas.height = height;
    }

    // draws bitmap to output (scaled)
    function drawBitmap2d(output, img, ctx) {
        const width = output.width || output.videoWidth;
        const height = output.height || output.videoHeight;
        ctx = ctx || output.getContext("2d");
        if (width == 0 || height == 0) { return; }
        ctx.clearRect(0, 0, width, height);
        ctx.drawImage(img, 0, 0, width, height);
    }

    // draws a color gradient using a 2d context
    function drawColor2d(output, color, ctx) {
        const width = output.width || output.videoWidth;
        const height = output.height || output.videoHeight;
        ctx = ctx || output.getContext("2d");
        if (width == 0 || height == 0) { return; }
        const gradient = ctx.createRadialGradient(width / 2, height / 2, 0, width / 2, height / 2, width);
        gradient.addColorStop(0, color);
        gradient.addColorStop(1, "black");
        ctx.fillStyle = gradient;
        ctx.fillRect(0, 0, width, height);
    }

    function useCpuBlur(canvas, radius = 8) {
        const ctx = canvas.getContext("2d", { willReadFrequently: true })
        const width = canvas.width;
        const height = canvas.height;
        const valueCount = radius * 2 + 1;
        {   // horizontal blurring
            const frame = ctx.getImageData(0, 0, width, height);
            const data = frame.data;
            const temp = new Uint8ClampedArray(data);
            for (let y = 0; y < height; y++) {
                let r = 0, g = 0, b = 0, a = 0;
                for (let x = -radius; x <= radius; x++) {
                    const clampedX = Math.min(width - 1, Math.max(0, x));
                    const idx = (y * width + clampedX) * 4;
                    r += temp[idx];
                    g += temp[idx + 1];
                    b += temp[idx + 2];
                    a += temp[idx + 3];
                }
                for (let x = 0; x < width; x++) {
                    const idx = (y * width + x) * 4;
                    data[idx] = r / valueCount;
                    data[idx + 1] = g / valueCount;
                    data[idx + 2] = b / valueCount;
                    data[idx + 3] = a / valueCount;
                    const remX = Math.max(0, x - radius);
                    const addX = Math.min(width - 1, x + radius + 1);
                    const remIdx = (y * width + remX) * 4;
                    const addIdx = (y * width + addX) * 4;
                    r += temp[addIdx] - temp[remIdx];
                    g += temp[addIdx + 1] - temp[remIdx + 1];
                    b += temp[addIdx + 2] - temp[remIdx + 2];
                    a += temp[addIdx + 3] - temp[remIdx + 3];
                }
            }
            ctx.putImageData(frame, 0, 0);
        }
        {   // vertical blurring
            const frame = ctx.getImageData(0, 0, width, height);
            const data = frame.data;
            const temp = new Uint8ClampedArray(data);
            for (let x = 0; x < width; x++) {
                let r = 0, g = 0, b = 0, a = 0;
                for (let y = -radius; y <= radius; y++) {
                    const clampedY = Math.min(height - 1, Math.max(0, y));
                    const idx = (clampedY * width + x) * 4;
                    r += temp[idx];
                    g += temp[idx + 1];
                    b += temp[idx + 2];
                    a += temp[idx + 3];
                }
                for (let y = 0; y < height; y++) {
                    const idx = (y * width + x) * 4;
                    data[idx] = r / valueCount;
                    data[idx + 1] = g / valueCount;
                    data[idx + 2] = b / valueCount;
                    data[idx + 3] = a / valueCount;
                    const remY = Math.max(0, y - radius);
                    const addY = Math.min(height - 1, y + radius + 1);
                    const remIdx = (remY * width + x) * 4;
                    const addIdx = (addY * width + x) * 4;
                    r += temp[addIdx] - temp[remIdx];
                    g += temp[addIdx + 1] - temp[remIdx + 1];
                    b += temp[addIdx + 2] - temp[remIdx + 2];
                    a += temp[addIdx + 3] - temp[remIdx + 3];
                }
            }
            ctx.putImageData(frame, 0, 0);
        }
    }

    function useCpuGrayscale(canvas) {
        const ctx = canvas.getContext("2d", { willReadFrequently: true })
        const width = canvas.width;
        const height = canvas.height;
        const frame = ctx.getImageData(0, 0, width, height);
        const data = frame.data;
        for (let idx = 0; idx < data.length; idx += 4) {
            // luminosity method (weighted average color)
            const c = 0.21 * data[idx] + 0.72 * data[idx + 1] + 0.07 * data[idx + 2];
            data[idx] = c;
            data[idx + 1] = c;
            data[idx + 2] = c;
        }
        ctx.putImageData(frame, 0, 0);
    }

    function useCpuSepia(canvas) {
        const ctx = canvas.getContext("2d", { willReadFrequently: true })
        const width = canvas.width;
        const height = canvas.height;
        const frame = ctx.getImageData(0, 0, width, height);
        const data = frame.data;
        for (let idx = 0; idx < data.length; idx += 4) {
            const red = data[idx];
            const green = data[idx + 1];
            const blue = data[idx + 2];
            const cr = Math.min(0.393 * red + 0.769 * green + 0.189 * blue, 255);
            const cg = Math.min(0.349 * red + 0.686 * green + 0.168 * blue, 255);
            const cb = Math.min(0.272 * red + 0.534 * green + 0.131 * blue, 255);
            data[idx] = cr;
            data[idx + 1] = cg;
            data[idx + 2] = cb;
        }
        ctx.putImageData(frame, 0, 0);
    }

    // implements video processing
    async function processVideo(obj) {
        if (!obj.video) {
            return;
        }

        // dimensions
        let videoWidth = obj.video.videoWidth;
        let videoHeight = obj.video.videoHeight;

        // buffer canvas context
        const ctx = obj.bCanvas.getContext("2d");

        // qCanvas
        let qCanvasWidth = obj.qCanvas.width;       // save width used to prevent false results while changing width during a calculation
        let qCanvasHeight = obj.qCanvas.height;     // save height used to prevent false results while changing height during a calculation

        // fgCanvas
        const fgCanvas = createCanvas(videoWidth, videoHeight);
        const fgCtx = fgCanvas.getContext("2d", { willReadFrequently: true });

        // bgCanvas in case that filter-property and webgl is not supported or active
        const bgCanvas = createCanvas(videoWidth, videoHeight);
        const bgCtx = bgCanvas.getContext("2d", { willReadFrequently: true });

        // time measurements (required for calling next frames)
        let timestamp;
        let delay;

        // buffer for mask data
        let previousMaskSize = 0;
        let previousMasks = [];

        // timeout for quality adjustments
        const adjustQualityTimeout = 5000;
        let time = 0;
        let timeout = 1000 / 60;

        // start process
        if (isSegmentationRequired()) {
            obj.currentState = INITIALIZE_MODULES;
            console.log("VirtualBackground::processVideo state update: " + "to INITIALIZE_MODULES");
        } else {
            obj.currentState = SEGMENTATION_NOT_REQUIRED;
            console.log("VirtualBackground::processVideo state update: " + "to SEGMENTATION_NOT_REQUIRED");
        }
        doNextFrame();      // starts video frame processing
        adjustQuality();    // starts auto quality adjustments

        // check if segmentation is required if not you should not do segmentation
        function isSegmentationRequired() {
            return obj.config.bmode != bmode.STREAM || obj.config.blur != 0 || obj.config.sepia || obj.config.grayscale;
        }

        // check if filter is required
        function isFilterRequired() {
            return obj.config.blur != 0 || obj.config.sepia || obj.config.grayscale;
        }

        // start next frame
        function doNextFrame() {
            timestamp = Date.now();
            // resize if required (handle video stream width / height changes)
            if (videoWidth != obj.video.videoWidth || videoHeight != obj.video.videoWidth) {
                let customSize = obj.canvas.width != videoWidth || obj.canvas.height != videoHeight;
                videoWidth = obj.video.videoWidth;
                videoHeight = obj.video.videoHeight;
                if (!customSize) resizeCanvas(obj.canvas, videoWidth, videoHeight);
                resizeCanvas(obj.bCanvas, videoWidth, videoHeight);
                if (obj.qmode == qmode.MAX) resizeCanvas(obj.qCanvas, videoWidth, videoHeight); 
                resizeCanvas(fgCanvas, videoWidth, videoHeight);
                resizeCanvas(bgCanvas, videoWidth, videoHeight);
            }
            try {
                switch (obj.currentState) {
                    case INITIALIZE_MODULES:
                    case LOAD_MODULES:
                    case INITIALIZE_SEGMENTER:
                    case LOAD_SEGMENTER:
                    case INITIALIZATION_FAILED:
                    case SEGMENTATION_NOT_REQUIRED:
                    case SEGMENTATION_NOT_POSSIBLE:
                        processFrame(obj.video);
                        break;
                    case SEGMENTATION:
                        if (obj.config.qmode != qmode.MAX) {
                            qCanvasWidth = obj.qCanvas.width;
                            qCanvasHeight = obj.qCanvas.height;
                            drawImage2d(obj.qCanvas, obj.video);
                            segmenter[obj.simd][obj.config.cmode].segmentForVideo(obj.qCanvas, performance.now(), processFrame);
                        } else {
                            segmenter[obj.simd][obj.config.cmode].segmentForVideo(obj.video, performance.now(), processFrame);
                        }
                        break;
                }
            } catch (err) {
                console.error("VirtualBackground::processVideo::doNextFrame failed: " + err);
                obj.currentState = SEGMENTATION_NOT_POSSIBLE; // in case segmentation doesn't work, just return the video.
                console.log("VirtualBackground::processVideo::doNextFrame state update: " + "to SEGMENTATION_NOT_POSSIBLE");
                obj.handleErrorInternal("Processing frame failed");
                processFrame(obj.video);
            }
        }

        // handle next frame while limiting fps if fps is set via worker
        if (obj.worker) {
            obj.worker.onmessage = function (e) {
                doNextFrame();
            }
        }

        // handle next frame while limiting fps if fps is set via main thread
        function handleNextFrame() {
            const fps = obj.config.fps
            delay = Date.now() - timestamp;
            if (fps > 0) {
                timeout = 1000 / fps;
            } else {
                timeout = 1000 / 60;
            }
            time = timeout - delay;
            if (obj.state == TERMINATED) {
                return;
            }
            if (window.Worker) {
                if (obj.worker) obj.worker.postMessage([time]);
            } else {
                obj.requestId = setTimeout(doNextFrame, time);
            }
        }

        // adjust quality loop
        function adjustQuality() {
            if (obj.config.autoqmode) {
                let nextqmode;
                if (time < 0) {                                     // decrease quality HIGH -> LOW
                    if (obj.config.qmode < HIGHEST_QMODE) {
                        nextqmode = HIGHEST_QMODE;
                    } else {
                        nextqmode = Math.min(obj.config.qmode + 1, LOWEST_QMODE);
                    }
                } else if (time > timeout) {                        // increase quality LOW -> HIGH
                    if (obj.config.qmode < HIGHEST_QMODE) {
                        nextqmode = HIGHEST_QMODE;
                    } else {
                        nextqmode = Math.max(obj.config.qmode - 1, HIGHEST_QMODE);
                    }
                }
                obj.setQualityModeInternal(nextqmode);
            }
            obj.autoAdjustQuality = setTimeout(adjustQuality, adjustQualityTimeout);
        }

        // implements frame processing
        async function processFrame(video) {
            switch (obj.currentState) {
                case INITIALIZE_MODULES:
                    try {
                        if (!that.config) {
                            console.error("VirtualBackground failed: " + "config path is not defined");
                            throw Error("config path is not defined");
                        }
                        if (!isModuleLoaded()) {
                            (async function () {
                                baseUrl = that.config.substring(0, that.config.substring(0, that.config.length - 1).lastIndexOf('/')) + "/";
                                vb = await import(that.config);
                                if (!vision) {
                                    (async function () {
                                        vision = await import(baseUrl + vb.config.libs.visionBundle);
                                    })();
                                }
                                if (!glfx) {
                                    let glfxElement = document.createElement("script");
                                    glfxElement.type = "text/javascript";
                                    glfxElement.src = baseUrl + vb.config.libs.glfx;
                                    glfxElement.onload = () => {
                                        glfx = true;
                                    }
                                    document.head.appendChild(glfxElement);
                                }
                                obj.currentState = LOAD_MODULES;
                                console.log("VirtualBackground::processVideo::processFrame state update: " + "from INITIALIZE_MODULES to LOAD_MODULES");
                            })();
                        } else {
                            obj.currentState = INITIALIZE_SEGMENTER;
                            console.log("VirtualBackground::processVideo::processFrame state update: " + "from LOAD_MODULES to INITIALIZE_SEGMENTER");
                        }
                    } catch (e) {
                        console.error("VirtualBackground::processVideo::processFrame failed: " + "failed to load modules, falling back video input stream");
                        obj.currentState = INITIALIZATION_FAILED;
                        console.log("VirtualBackground::processVideo::processFrame state update: " + "from INITIALIZE_MODULES to INITIALIZATION_FAILED");
                        obj.handleErrorInternal("Initialization failed (modules)");
                    }
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
                case LOAD_MODULES:
                    if (isModuleLoaded()) {
                        obj.currentState = INITIALIZE_SEGMENTER;
                        console.log("VirtualBackground::processVideo::processFrame state update: " + "from LOAD_MODULES to INITIALIZE_SEGMENTER");
                    }
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
                case INITIALIZE_SEGMENTER:
                    try {
                        obj.simd = await vision.FilesetResolver.isSimdSupported() ? "simd" : "nosimd";
                        if (!segmenter[obj.simd][obj.config.cmode]) {
                            const wasmFileset = {
                                wasmLoaderPath: baseUrl + vb.config.wasm[obj.simd].js,
                                wasmBinaryPath: baseUrl + vb.config.wasm[obj.simd].file
                            }
                            segmenter[obj.simd][obj.config.cmode] = await vision.ImageSegmenter.createFromOptions(wasmFileset, {
                                baseOptions: {
                                    modelAssetPath: baseUrl + vb.config.model,
                                    delegate: obj.config.cmode
                                },
                                outputCategoryMask: false, // required for category mask usage
                                outputConfidenceMasks: true, // required for confidence mask usage
                                runningMode: "VIDEO"
                            });
                        }
                        obj.currentState = LOAD_SEGMENTER;
                        console.log("VirtualBackground::processVideo::processFrame state update: " + "from INITIALIZE_SEGMENTER to LOAD_SEGMENTER");
                    } catch (e) {
                        console.error("VirtualBackground::processVideo::processFrame failed: " + "failed to load segmenter, falling back video input stream");
                        obj.currentState = INITIALIZATION_FAILED;
                        console.log("VirtualBackground::processVideo::processFrame state update: " + "from INITIALIZE_SEGMENTER to INITIALIZATION_FAILED");
                        obj.handleErrorInternal("Initialization failed (segmenter)");
                    }
                    try {
                        obj.glfx = {};
                        obj.glfx.canvas = fx.canvas();
                        obj.glfx.streamTexture = obj.glfx.canvas.texture(obj.video);
                    } catch (e) {
                        console.warn("VirtualBackground::processVideo::processFrame failed: " + "failed to initialize glfx using webgl (" + e + ")");
                        obj.glfx = null;
                    }
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
                case LOAD_SEGMENTER:
                    if (segmenter[obj.simd][obj.config.cmode]) {
                        if (isSegmentationRequired()) {
                            obj.currentState = SEGMENTATION;
                            console.log("VirtualBackground::processVideo::processFrame state update: " + "from LOAD_SEGMENTER to SEGMENTATION");
                        } else {
                            obj.currentState = SEGMENTATION_NOT_REQUIRED;
                            console.log("VirtualBackground::processVideo::processFrame state update: " + "from LOAD_SEGMENTER to SEGMENTATION_NOT_REQUIRED");
                        }
                    }
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
                case SEGMENTATION:
                    try {
                        if (!isSegmentationRequired()) {
                            obj.currentState = SEGMENTATION_NOT_REQUIRED;
                            console.log("VirtualBackground::processVideo::processFrame state update: " + "from SEGMENTATION to SEGMENTATION_NOT_REQUIRED");
                        }
                        // mask
                        const maskData = video.confidenceMasks[0].getAsFloat32Array(); // using confidence mask
                        //const maskData = video.categoryMask.getAsUint8Array(); // using category mask 
                        const maskSize = maskData.length;
                        let maskBuffer = new Uint8ClampedArray(4 * maskSize);
                        if (previousMaskSize != maskSize) {
                            console.log("VirtualBackground::processVideo::processFrame cache reset: " + "mask size do not match " + JSON.stringify({ previousSize: previousMaskSize, newSize: maskSize }));
                            previousMasks = [];
                        }
                        let i = 0;
                        let j = 0;
                        let len = previousMasks.length;
                        while (i < maskSize) {
                            let k = 0;
                            maskBuffer[j + 3] = maskData[i] * 255 / (len + 1);
                            while (k < len) {
                                maskBuffer[j + 3] += previousMasks[k][j + 3] / (len + 1);
                                k++;
                            }
                            i++;
                            j += 4;
                        }
                        previousMaskSize = maskSize;
                        previousMasks.push(maskBuffer);
                        while (previousMasks.length > obj.config.smoothing) {
                            previousMasks.shift();
                        }

                        // draw foreground (safari has problems with globalCompositeOperation source-in so it uses a seperate way)
                        if (typeof (ctx.filter) !== 'undefined') {    // not supported on safari browser
                            // draw foreground (default)
                            const mask = videoWidth * videoHeight == maskSize ? await createImageBitmap(new ImageData(maskBuffer, videoWidth, videoHeight)) : await createImageBitmap(new ImageData(maskBuffer, qCanvasWidth, qCanvasHeight));
                            ctx.filter = "blur(" + MASK_BLUR_RADIUS + "px)"
                            drawBitmap2d(obj.bCanvas, mask);
                            ctx.filter = "none";
                            mask.close();
                            // overwrite segmentation mask (foreground)
                            ctx.globalCompositeOperation = "source-in";
                            drawImage2d(obj.bCanvas, obj.video);
                        } else {
                            // draw foreground (safari and all browser that do not support filter)
                            drawImage2d(fgCanvas, obj.video);
                            const frame = fgCtx.getImageData(0, 0, videoWidth, videoHeight);
                            const data = frame.data;
                            const mask = videoWidth * videoHeight == maskSize ? await createImageBitmap(new ImageData(maskBuffer, videoWidth, videoHeight)) : await createImageBitmap(new ImageData(maskBuffer, qCanvasWidth, qCanvasHeight));
                            drawBitmap2d(fgCanvas, mask);
                            mask.close();
                            //useCpuBlur(fgCanvas, MASK_BLUR_RADIUS); // currently not using cpu blur (reduces too much performance)
                            const maskFrame = fgCtx.getImageData(0, 0, videoWidth, videoHeight);
                            const maskDataFrame = maskFrame.data;
                            for (let idx = 0; idx < data.length; idx += 4) {
                                data[idx + 3] = maskDataFrame[idx + 3];
                            }
                            fgCtx.putImageData(frame, 0, 0);
                            ctx.clearRect(0, 0, videoWidth, videoHeight);
                            drawImage2d(obj.bCanvas, fgCanvas);
                        }

                        // overwrite missing pixels (background)
                        ctx.globalCompositeOperation = "destination-over";
                        switch (obj.config.bmode) {
                            case bmode.STREAM:
                                if (typeof (ctx.filter) !== 'undefined') {    // not supported on safari browser
                                    ctx.filter = "blur(" + obj.config.blur + "px)"
                                    if (obj.config.grayscale) {
                                        ctx.filter += " grayscale()";
                                    } else if (obj.config.sepia) {
                                        ctx.filter += " sepia()";
                                    }
                                    drawImage2d(obj.bCanvas, obj.video);
                                    ctx.filter = "none";
                                }
                                else if (obj.glfx) {                        // uses webgl (for safari browser)
                                    obj.glfx.streamTexture.loadContentsOf(obj.video);
                                    obj.glfx.canvas.draw(obj.glfx.streamTexture)
                                        .lensBlur(obj.config.blur, 0, 0)
                                        .hueSaturation(0, obj.config.grayscale ? -1 : 0)
                                        .sepia(obj.config.sepia ? 1 : 0)
                                        .update();
                                    drawImage2d(obj.bCanvas, obj.glfx.canvas);
                                }
                                else {                                      // manually calculate pixel color efficiently using sliding windows
                                    if (isFilterRequired()) {
                                        drawImage2d(bgCanvas, obj.video);
                                        if (obj.config.blur != 0) { // blurring
                                            useCpuBlur(bgCanvas, obj.config.blur);
                                        }
                                        if (obj.config.grayscale) {
                                            useCpuGrayscale(bgCanvas);
                                        }
                                        else if (obj.config.sepia) {
                                            useCpuSepia(bgCanvas);
                                        }
                                        drawImage2d(obj.bCanvas, bgCanvas);
                                    } else {
                                        drawImage2d(obj.bCanvas, obj.video);
                                    }
                                }
                                break;
                            case bmode.COLOR:
                                drawColor2d(obj.bCanvas, obj.config.color);
                                break;
                            case bmode.IMAGE:
                                drawImage2d(obj.bCanvas, obj.config.image, obj.config.dmodeImage);
                                break;
                        }
                        ctx.globalCompositeOperation = "source-over";
                        drawImage2d(obj.canvas, obj.bCanvas, obj.config.dmode);
                    } catch (e) {
                        console.error("VirtualBackground::processVideo::processFrame failed: " + "failed to handle segmentation result, falling back video input stream");
                        obj.currentState = SEGMENTATION_NOT_POSSIBLE;
                        console.log("VirtualBackground::processVideo::processFrame state update: " + "from SEGMENTATION to SEGMENTATION_NOT_POSSIBLE");
                        obj.handleErrorInternal("Handling segmentation failed");
                        ctx.globalCompositeOperation = "source-over";
                        drawImage2d(obj.canvas, obj.video, obj.config.dmode);
                    }
                    break;
                case SEGMENTATION_NOT_REQUIRED:
                    if (isSegmentationRequired()) {
                        if (obj.simd && segmenter[obj.simd][obj.config.cmode]) {
                            obj.currentState = SEGMENTATION;
                            console.log("VirtualBackground::processVideo::processFrame state update: " + "from SEGMENTATION_NOT_REQUIRED to SEGMENTATION");
                        }
                        else {
                            if (!isModuleLoaded()) {
                                obj.currentState = INITIALIZE_MODULES;
                                console.log("VirtualBackground::processVideo::processFrame state update: " + "from SEGMENTATION_NOT_REQUIRED to INITIALIZE_MODULES");
                            } else {
                                obj.currentState = INITIALIZE_SEGMENTER;
                                console.log("VirtualBackground::processVideo::processFrame state update: " + "from SEGMENTATION_NOT_REQUIRED to INITIALIZE_SEGMENTER");
                            }
                        }
                    }
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
                case INITIALIZATION_FAILED:
                case SEGMENTATION_NOT_POSSIBLE:
                    drawImage2d(obj.canvas, video, obj.config.dmode);
                    break;
            }
            handleNextFrame();
        }
    }
    console.log("VirtualBackground success: " + "virtualbackground initialized");
}
innovaphone.VirtualBackground.prototype = innovaphone.ui1.nodePrototype;