Feature/259/get screen pixel color (#325)

* (#259) Introduced ImageProcessor interface

* (#259) imageToJimp converter function

* (#259) RGBA dataclass

* (#259) Implement ImageProcessor and register the new provider

* (#259) Re-use imageToJimp and clean up tests

* (#259) Fixed import for Image which was previously importing from dist

* Maintenance/310/remove adapters (#311)

* (#310) Delete both adapters

* (#310) Migrate window.function to not use adapters

* (#310) Migrated window.class to not use adapters

* (#310) Migrated screen.class to not use adapters

* (#310) Migrated movement.function to not use adapters

* (#310) Migrated mouse.class to not use adapters

* (#310) Remove leftover notions of adapters in test description

* (#310) Fixed expected format when saving a screenshot to disk

* (#310) Migrated keyboard.class to not use adapters

* (#310) Migrated clipboard.class to not use adapters

* (#310) Migrated assert.class test to not use adapters

* (#310) Migrated all exported instances to not use adapters

* (#310) Removed mention of adapter from test description

* Added sponsors listing to README.md

* (#306) Update supported cpus to include Apple Silicon (arm64)

* Add https://github.com/stoefln to sponsors listing

* Feature/204/screen find image needles (#319)

* (#204) Enabled passing of Image data to Screen#find

* (#204) Updated screen tests

* (#204) Removed plugin test from main repo

* (#204) Removed plugin test from Docker test runs

* (#204) Introduced additional id property to images

* (#204) Adjusted tests and image usage to new id property

* (#204) Added helper function to make loading image resources easier

* (#204) Adapted toShow matcher to new find parameters

* (#204) Re-use loadImageResource function

* Feature/320/find accept promise (#322)

* (#320) Make find, and the functions re-using it, accept Promise<Image>

* (#320) Update toShow matcher accordingly

* (#320) Update tests

* (#321) Added `findAll` to Screen (#323)

* (#259) Add `colorAt` to screen class

Author: s1hofmann

e2e/assets/checkers.png

@@ -25,6 +25,7 @@ export * from "./lib/provider";
 export {jestMatchers} from "./lib/expect/jest.matcher.function";
 export {sleep} from "./lib/sleep.function";
 export {Image} from "./lib/image.class";
+export {RGBA} from "./lib/rgba.class";
 export {Key} from "./lib/key.enum";
 export {Button} from "./lib/button.enum";
 export {centerOf, randomPointIn} from "./lib/location.function";

index.ts

@@ -0,0 +1,7 @@
+import {Point} from "../point.class";
+import {RGBA} from "../rgba.class";
+import {Image} from "../image.class";
+
+export interface ImageProcessor {
+    colorAt(image: Image | Promise<Image>, location: Point | Promise<Point>): Promise<RGBA>;
+}

lib/provider/image-processor.interface.ts

@@ -0,0 +1,24 @@
+import Jimp from 'jimp';
+import {Image} from "../../image.class";
+import {Point} from "../../point.class";
+import {ImageProcessor} from "../image-processor.interface";
+import {imageToJimp} from "../io/imageToJimp.function";
+import {RGBA} from "../../rgba.class";
+
+export default class implements ImageProcessor {
+    colorAt(image: Image | Promise<Image>, point: Point | Promise<Point>): Promise<RGBA> {
+        return new Promise<RGBA>(async (resolve, reject) => {
+            const location = await point;
+            const img = await image;
+            if (location.x < 0 || location.x >= img.width) {
+                reject(`Query location out of bounds. Should be in range 0 <= x < image.width, is ${location.x}`);
+            }
+            if (location.y < 0 || location.y >= img.height) {
+                reject(`Query location out of bounds. Should be in range 0 <= y < image.height, is ${location.y}`);
+            }
+            const jimpImage = imageToJimp(img);
+            const rgba = Jimp.intToRGBA(jimpImage.getPixelColor(location.x, location.y));
+            resolve(new RGBA(rgba.r, rgba.g, rgba.b, rgba.a));
+        });
+    }
+}

lib/provider/image/jimp-image-processor.class.ts

@@ -0,0 +1,38 @@
+import {Image} from "../../image.class";
+import {imageToJimp} from "./imageToJimp.function";
+import Jimp from "jimp";
+
+jest.mock('jimp', () => {
+    class JimpMock {
+        bitmap = {
+            width: 100,
+            height: 100,
+            data: Buffer.from([]),
+        }
+        hasAlpha = () => false
+        static read = jest.fn(() => Promise.resolve(new JimpMock()))
+    }
+
+    return ({
+        __esModule: true,
+        default: JimpMock
+    })
+});
+
+afterEach(() => jest.resetAllMocks());
+
+describe('imageToJimp', () => {
+    it('should successfully convert an Image to a Jimp instance', async () => {
+        // GIVEN
+        const scanMock = jest.fn();
+        Jimp.prototype.scan = scanMock;
+        const inputImage = new Image(1, 1, Buffer.from([0, 0, 0]),3, "input_image");
+
+        // WHEN
+        const result = await imageToJimp(inputImage);
+
+        // THEN
+        expect(result).toBeInstanceOf(Jimp);
+        expect(scanMock).toHaveBeenCalledTimes(1);
+    });
+});

lib/provider/io/imageToJimp.function.spec.ts

@@ -0,0 +1,17 @@
+import Jimp from "jimp";
+import {Image} from "../../image.class";
+
+export function imageToJimp(image: Image): Jimp {
+    const jimpImage = new Jimp({
+        data: image.data,
+        width: image.width,
+        height: image.height
+    });
+    // Images treat data in BGR format, so we have to switch red and blue color channels
+    jimpImage.scan(0, 0, jimpImage.bitmap.width, jimpImage.bitmap.height, function (_, __, idx) {
+        const red = this.bitmap.data[idx];
+        this.bitmap.data[idx] = this.bitmap.data[idx + 2];
+        this.bitmap.data[idx + 2] = red;
+    });
+    return jimpImage;
+}

lib/provider/io/imageToJimp.function.ts

@@ -2,7 +2,6 @@ import ImageReader from "./jimp-image-reader.class";
 import {join} from "path";
 import Jimp from "jimp";
 
-jest.mock('gifwrap', () => {});
 jest.mock('jimp', () => {
     class JimpMock {
         bitmap = {

lib/provider/io/jimp-image-reader.class.spec.ts

@@ -2,7 +2,6 @@ import ImageWriter from "./jimp-image-writer.class";
 import {Image} from "../../image.class";
 import Jimp from "jimp";
 
-jest.mock('gifwrap', () => {});
 jest.mock('jimp', () => {
     class JimpMock {
         bitmap = {

lib/provider/io/jimp-image-writer.class.spec.ts

@@ -1,20 +1,10 @@
-import Jimp from 'jimp';
 import {ImageWriter, ImageWriterParameters} from "../image-writer.type";
+import {imageToJimp} from "./imageToJimp.function";
 
 export default class implements ImageWriter {
     store(parameters: ImageWriterParameters): Promise<void> {
         return new Promise((resolve, reject) => {
-            const jimpImage = new Jimp({
-                data: parameters.data.data,
-                width: parameters.data.width,
-                height: parameters.data.height
-            });
-            // libnut returns data in BGR format, so we have to switch red and blue color channels
-            jimpImage.scan(0, 0, jimpImage.bitmap.width, jimpImage.bitmap.height, function (_, __, idx) {
-                const red = this.bitmap.data[idx];
-                this.bitmap.data[idx] = this.bitmap.data[idx + 2];
-                this.bitmap.data[idx + 2] = red;
-            });
+            const jimpImage = imageToJimp(parameters.data);
             jimpImage
                 .writeAsync(parameters.path)
                 .then(_ => resolve())

lib/provider/io/jimp-image-writer.class.ts

@@ -12,34 +12,48 @@ import Screen from "./native/libnut-screen.class";
 import Window from "./native/libnut-window.class";
 import {ImageReader} from "./image-reader.type";
 import {ImageWriter} from "./image-writer.type";
+import {ImageProcessor} from "./image-processor.interface";
 
 import ImageReaderImpl from "./io/jimp-image-reader.class";
 import ImageWriterImpl from "./io/jimp-image-writer.class";
+import ImageProcessorImpl from "./image/jimp-image-processor.class";
 
 export interface ProviderRegistry {
     getClipboard(): ClipboardProviderInterface;
-    registerClipboardProvider(value: ClipboardProviderInterface): void;
 
-    getImageFinder(): ImageFinderInterface;
-    registerImageFinder(value: ImageFinderInterface): void;
+    registerClipboardProvider(value: ClipboardProviderInterface): void;
 
     getKeyboard(): KeyboardProviderInterface;
+
     registerKeyboardProvider(value: KeyboardProviderInterface): void;
 
     getMouse(): MouseProviderInterface;
+
     registerMouseProvider(value: MouseProviderInterface): void;
 
     getScreen(): ScreenProviderInterface;
+
     registerScreenProvider(value: ScreenProviderInterface): void;
 
     getWindow(): WindowProviderInterface;
+
     registerWindowProvider(value: WindowProviderInterface): void;
 
+    getImageFinder(): ImageFinderInterface;
+
+    registerImageFinder(value: ImageFinderInterface): void;
+
     getImageReader(): ImageReader;
+
     registerImageReader(value: ImageReader): void;
 
     getImageWriter(): ImageWriter;
+
     registerImageWriter(value: ImageWriter): void;
+
+    getImageProcessor(): ImageProcessor;
+
+    registerImageProcessor(value: ImageProcessor): void;
 }
 
 class DefaultProviderRegistry implements ProviderRegistry {
@@ -51,6 +65,7 @@ class DefaultProviderRegistry implements ProviderRegistry {
     private _window?: WindowProviderInterface;
     private _imageReader?: ImageReader;
     private _imageWriter?: ImageWriter;
+    private _imageProcessor?: ImageProcessor;
 
     getClipboard(): ClipboardProviderInterface {
         if (this._clipboard) {
@@ -124,6 +139,7 @@ class DefaultProviderRegistry implements ProviderRegistry {
         }
         throw new Error(`No ImageReader registered`);
     }
+
     registerImageReader(value: ImageReader) {
         this._imageReader = value;
     }
@@ -134,9 +150,21 @@ class DefaultProviderRegistry implements ProviderRegistry {
         }
         throw new Error(`No ImageWriter registered`);
     }
+
     registerImageWriter(value: ImageWriter) {
         this._imageWriter = value;
     }
+
+    getImageProcessor(): ImageProcessor {
+        if (this._imageProcessor) {
+            return this._imageProcessor;
+        }
+        throw new Error(`No ImageProcessor registered`);
+    }
+
+    registerImageProcessor(value: ImageProcessor): void {
+        this._imageProcessor = value;
+    }
 }
 
 const providerRegistry = new DefaultProviderRegistry();
@@ -148,5 +176,6 @@ providerRegistry.registerScreenProvider(new Screen());
 providerRegistry.registerWindowProvider(new Window());
 providerRegistry.registerImageWriter(new ImageWriterImpl());
 providerRegistry.registerImageReader(new ImageReaderImpl());
+providerRegistry.registerImageProcessor(new ImageProcessorImpl());
 
 export default providerRegistry;

lib/provider/provider-registry.class.ts

@@ -0,0 +1,13 @@
+export class RGBA {
+    constructor(public readonly R: number, public readonly G: number, public readonly B: number, public readonly A: number) {
+    }
+
+    public toString(): string {
+        return `rgb(${this.R},${this.G},${this.B})`;
+    }
+
+    public toHex(): string {
+        return `#${this.R.toString(16).padStart(2, '0')}${this.G.toString(16).padStart(2, '0')}${this.B.toString(16).padStart(2, '0')}${this.A.toString(16).padStart(2, '0')}`
+    }
+}
+

lib/rgba.class.ts

@@ -10,6 +10,7 @@ import {Image} from "./image.class";
 import {ProviderRegistry} from "./provider/provider-registry.class";
 import {loadImageResource} from "./imageResources.function";
 import {FirstArgumentType} from "./typings";
+import {Point} from "./point.class";
 
 export type FindHookCallback = (target: MatchResult) => Promise<void>;
 
@@ -308,6 +309,15 @@ export class ScreenClass {
         return this.providerRegistry.getScreen().grabScreenRegion(await regionToGrab);
     }
 
+    /**
+     * {@link colorAt} returns RGBA color values for a certain pixel at {@link Point} p
+     * @param point Location to query color information from
+     */
+    public async colorAt(point: Point | Promise<Point>) {
+        const screenContent = this.providerRegistry.getScreen().grabScreen();
+        return this.providerRegistry.getImageProcessor().colorAt(screenContent, point);
+    }
+
     private async saveImage(
         image: Image,
         fileName: string,

lib/screen.class.ts

@@ -0,0 +1,44 @@
+import {Image, loadImage, Point, Region, RGBA, ScreenClass, ScreenProviderInterface} from "../index";
+import {mockPartial} from "sneer";
+import providerRegistry, {ProviderRegistry} from "./provider/provider-registry.class";
+import {ImageProcessor} from "./provider/image-processor.interface";
+
+const searchRegion = new Region(0, 0, 1000, 1000);
+const providerRegistryMock = mockPartial<ProviderRegistry>({
+    getScreen(): ScreenProviderInterface {
+        return mockPartial<ScreenProviderInterface>({
+            grabScreenRegion(): Promise<Image> {
+                return Promise.resolve(new Image(searchRegion.width, searchRegion.height, new ArrayBuffer(0), 3, "needle_image"));
+            },
+            screenSize(): Promise<Region> {
+                return Promise.resolve(searchRegion);
+            }
+        })
+    },
+    getImageProcessor(): ImageProcessor {
+        return providerRegistry.getImageProcessor();
+    }
+});
+
+describe("colorAt", () => {
+    it("should return the correct RGBA value for a given pixel", async () => {
+        // GIVEN
+        const screenshot = loadImage(`${__dirname}/../e2e/assets/checkers.png`);
+        const grabScreenMock = jest.fn(() => Promise.resolve(screenshot));
+        providerRegistryMock.getScreen = jest.fn(() => mockPartial<ScreenProviderInterface>({
+            grabScreen: grabScreenMock
+        }));
+        providerRegistryMock.getImageProcessor()
+        const SUT = new ScreenClass(providerRegistryMock);
+        const expectedWhite = new RGBA(255, 255, 255, 255);
+        const expectedBlack = new RGBA(0, 0, 0, 255);
+
+        // WHEN
+        const white = await SUT.colorAt(new Point(64, 64));
+        const black = await SUT.colorAt(new Point(192, 64));
+
+        // THEN
+        expect(white).toStrictEqual(expectedWhite);
+        expect(black).toStrictEqual(expectedBlack);
+    });
+});