From 94cbe8999d573e9002d0f1a24fc01a9a2d032648 Mon Sep 17 00:00:00 2001
From: Christopher Willis-Ford <cwillisf@media.mit.edu>
Date: Fri, 27 May 2016 16:10:22 -0700
Subject: [PATCH] Implement pick(x,y) -> Drawable ID

---
 build/demo.html         |  8 ++--
 src/Drawable.js         | 63 ++++++++++++++++++++++----
 src/index.js            | 97 +++++++++++++++++++++++++++++++++++++++--
 src/shaders/sprite.frag | 44 ++++++++++++-------
 4 files changed, 181 insertions(+), 31 deletions(-)
diff --git a/build/demo.html b/build/demo.html
index acf5ab9..a33f203 100644
--- a/build/demo.html
+++ b/build/demo.html
@@ -7,7 +7,7 @@
 <body>
 <canvas id="scratch-stage" width="10" height="10" style="border:1px dashed black; margin-top: 9px;"></canvas>
 <p>
-    <input type="range" id="fudge" style="width:50%" min="0" max="100" step="any" oninput="onFudgeChanged(this.value)" onchange="onFudgeChanged(this.value)">
+    <input type="range" id="fudge" style="width:50%" value="90" min="-90" max="270" step="any" oninput="onFudgeChanged(this.value)" onchange="onFudgeChanged(this.value)">
 </p>
 <p>
     Min: <input id="fudgeMin" type="number" onchange="onFudgeMinChanged(this.value)">
@@ -16,7 +16,7 @@
 </body>
 <script src="render-webgl.js"></script>
 <script>
-    window.fudge = 0;
+    window.fudge = 90;
     var fudgeInput = document.getElementById('fudge');
     function onFudgeMinChanged(newValue) {
         fudgeInput.min = newValue;
@@ -47,8 +47,8 @@
 
         var props = {};
         //props.position = [posX, posY];
-        props.direction = direction;
-        props.color = fudge;
+        props.direction = fudge;
+        //props.pixelate = fudge;
         //props.scale = 100;
 
         renderer.updateDrawableProperties(drawableID, props);
diff --git a/src/Drawable.js b/src/Drawable.js
index 5535b67..de9794a 100644
--- a/src/Drawable.js
+++ b/src/Drawable.js
@@ -95,18 +95,29 @@ Drawable._effectConverter = {
     }
 };
 
+/**
+ * An invalid Drawable ID which can be used to signify absence, etc.
+ * @type {int}
+ */
+Drawable.NONE = -1;
+
 /**
  * The name of each supported effect.
  * @type {Array}
  */
 Drawable.EFFECTS = Object.keys(Drawable._effectConverter);
 
+Drawable.DRAW_MODE = {
+    default: 'default',
+    pick: 'pick'
+};
+
 /**
  * The cache of all shaders compiled so far. These are generated on demand.
- * @type {Array}
+ * @type {Object.<Drawable.DRAW_MODE, Array.<module:twgl.ProgramInfo>>}
  * @private
  */
-Drawable._shaderCache = [];
+Drawable._shaderCache = {};
 
 /**
  * The ID to be assigned next time the Drawable constructor is called.
@@ -195,26 +206,33 @@ Drawable.prototype.setSkin = function (skin_md5ext) {
 /**
  * Fetch the shader for this Drawable's set of active effects.
  * Build the shader if necessary.
+ * @param {Drawable.DRAW_MODE} drawMode Draw normally or for picking, etc.
  * @returns {module:twgl.ProgramInfo?} The shader's program info.
  */
-Drawable.prototype.getShader = function () {
-    var shader = Drawable._shaderCache[this._shaderIndex];
+Drawable.prototype.getShader = function (drawMode) {
+    var cache = Drawable._shaderCache[drawMode];
+    if (!cache) {
+        cache = Drawable._shaderCache[drawMode] = [];
+    }
+    var shader = cache[this._shaderIndex];
     if (!shader) {
-        shader = Drawable._shaderCache[this._shaderIndex] =
-            this._buildShader();
+        shader = cache[this._shaderIndex] = this._buildShader(drawMode);
     }
     return shader;
 };
 
 /**
  * Build the shader for this Drawable's set of active effects.
+ * @param {Drawable.DRAW_MODE} drawMode Draw normally or for picking, etc.
  * @returns {module:twgl.ProgramInfo?} The new shader's program info.
  * @private
  */
-Drawable.prototype._buildShader = function () {
-    var defines = [];
+Drawable.prototype._buildShader = function (drawMode) {
     var numEffects = Drawable.EFFECTS.length;
 
+    var defines = [
+        '#define DRAW_MODE_' + drawMode
+    ];
     for (var index = 0; index < numEffects; ++index) {
         if ((this._shaderIndex & (1 << index)) != 0) {
             defines.push('#define ENABLE_' + Drawable.EFFECTS[index]);
@@ -405,3 +423,32 @@ Drawable.prototype._calculateTransform = function () {
 
     this._transformDirty = false;
 };
+
+/**
+ * Calculate a color to represent the given ID number. At least one component of
+ * the resulting color will be non-zero if the ID is not Drawable.NONE.
+ * @param {int} id The ID to convert.
+ * @returns {number[]} An array of [r,g,b,a], each component in the range [0,1].
+ */
+Drawable.color4fFromID = function(id) {
+    id -= Drawable.NONE;
+    var r = ((id >> 0) & 255) / 255.0;
+    var g = ((id >> 8) & 255) / 255.0;
+    var b = ((id >> 16) & 255) / 255.0;
+    return [r, g, b, 1.0];
+};
+
+/**
+ * Calculate the ID number represented by the given color. If all components of
+ * the color are zero, the result will be Drawable.NONE; otherwise the result
+ * will be a valid ID.
+ * @param {Array.<int>} rgba An array of [r,g,b,a], each component a byte.
+ * @returns {int} The ID represented by that color.
+ */
+Drawable.color4ubToID = function(rgba) {
+    var id;
+    id = (rgba[0] & 255) << 0;
+    id |= (rgba[1] & 255) << 8;
+    id |= (rgba[2] & 255) << 16;
+    return id + Drawable.NONE;
+};
diff --git a/src/index.js b/src/index.js
index 7113196..a647271 100644
--- a/src/index.js
+++ b/src/index.js
@@ -41,6 +41,7 @@ function RenderWebGL(
     this.resize(
         pixelsWide || Math.abs(this._xRight - this._xLeft),
         pixelsTall || Math.abs(this._yTop - this._yBottom));
+    this._createQueryBuffers();
 }
 
 /**
@@ -99,6 +100,7 @@ RenderWebGL.prototype.resize = function (pixelsWide, pixelsTall) {
 RenderWebGL.prototype.draw = function () {
     var gl = this._gl;
 
+    twgl.bindFramebufferInfo(gl, null);
     gl.viewport(0, 0, gl.canvas.clientWidth, gl.canvas.clientHeight);
     gl.clearColor.apply(gl, this._backgroundColor);
     gl.clear(gl.COLOR_BUFFER_BIT);
@@ -106,22 +108,45 @@ RenderWebGL.prototype.draw = function () {
     gl.enable(gl.BLEND);
     gl.blendFuncSeparate(gl.ONE, gl.ONE_MINUS_SRC_ALPHA, gl.ZERO, gl.ONE);
 
+    this._drawExcept(Drawable.DRAW_MODE.default, null, this._projection);
+};
+
+/**
+ * Draw all Drawables, with the possible exception of
+ * @param {Drawable.DRAW_MODE} drawMode Draw normally or for picking, etc.
+ * @param {int} skipID The Drawable to skip, if any.
+ * @param {module:twgl/m4.Mat4} projection The projection matrix to use.
+ * @private
+ */
+RenderWebGL.prototype._drawExcept = function(drawMode, skipID, projection) {
+    var gl = this._gl;
     var currentShader = null;
 
     var numDrawables = this._drawables.length;
     for (var drawableIndex = 0; drawableIndex < numDrawables; ++drawableIndex) {
         var drawableID = this._drawables[drawableIndex];
+        if (drawableID == skipID) continue;
+
         var drawable = Drawable.getDrawableByID(drawableID);
-        var newShader = drawable.getShader();
+        // TODO: check if drawable is inside the viewport before anything else
+
+        var newShader = drawable.getShader(drawMode);
         if (currentShader != newShader) {
             currentShader = newShader;
             gl.useProgram(currentShader.program);
             twgl.setBuffersAndAttributes(gl, currentShader, this._bufferInfo);
-            twgl.setUniforms(
-                currentShader, {u_projectionMatrix: this._projection});
+            twgl.setUniforms(currentShader, {u_projectionMatrix: projection});
             twgl.setUniforms(currentShader, {u_fudge: window.fudge || 0});
         }
+
         twgl.setUniforms(currentShader, drawable.getUniforms());
+
+        // TODO: consider moving u_pickColor into Drawable's getUniforms()...
+        if (drawMode == Drawable.DRAW_MODE.pick) {
+            twgl.setUniforms(currentShader,
+                {u_pickColor: Drawable.color4fFromID(drawableID)});
+        }
+
         twgl.drawBufferInfo(gl, gl.TRIANGLES, this._bufferInfo);
     }
 };
@@ -205,3 +230,69 @@ RenderWebGL.prototype._createGeometry = function () {
     };
     this._bufferInfo = twgl.createBufferInfoFromArrays(this._gl, quad);
 };
+
+/**
+ * Create the frame buffers used for queries such as picking and color-touching.
+ * These buffers are fixed in size regardless of the size of the main render
+ * target. The fixed size allows (more) consistent behavior across devices and
+ * presentation modes.
+ * @private
+ */
+RenderWebGL.prototype._createQueryBuffers = function () {
+    var gl = this._gl;
+    var attachments = [
+        {format: gl.RGBA },
+        {format: gl.DEPTH_STENCIL }
+    ];
+
+    // TODO: consider larger sizes for multi-sample touch picking
+    var pickBufferWidth = 1;
+    var pickBufferHeight = 1;
+
+    this._pickBufferInfo = twgl.createFramebufferInfo(
+        gl, attachments, pickBufferWidth, pickBufferHeight);
+};
+
+/**
+ * @param {int} centerX The canvas x coordinate of the picking location.
+ * @param {int} centerY The canvas y coordinate of the picking location.
+ * @returns {int} The ID of the topmost Drawable under the picking location, or
+ * Drawable.NONE if there is no Drawable at that location.
+ */
+RenderWebGL.prototype.pick = function (centerX, centerY) {
+    var gl = this._gl;
+
+    // TODO: consider larger sizes for multi-sample touch picking
+    var touchWidth = 1;
+    var touchHeight = 1;
+    var pixelLeft = centerX - Math.floor(touchWidth / 2);
+    var pixelRight = centerX + Math.ceil(touchWidth / 2);
+    var pixelTop = centerY - Math.floor(touchHeight / 2);
+    var pixelBottom = centerY + Math.ceil(touchHeight / 2);
+
+    twgl.bindFramebufferInfo(gl, this._pickBufferInfo);
+    gl.viewport(0, 0, touchWidth, touchHeight);
+    gl.disable(gl.BLEND);
+
+    var noneColor = Drawable.color4fFromID(Drawable.NONE);
+    gl.clearColor.apply(gl, noneColor);
+    gl.clear(gl.COLOR_BUFFER_BIT);
+
+    var widthPerPixel = (this._xRight - this._xLeft) / this._gl.canvas.width;
+    var heightPerPixel = (this._yBottom - this._yTop) / this._gl.canvas.height;
+
+    var pickLeft = this._xLeft + pixelLeft * widthPerPixel;
+    var pickRight = this._xLeft + pixelRight * widthPerPixel;
+    var pickTop = this._yTop + pixelTop * heightPerPixel;
+    var pickBottom = this._yTop + pixelBottom * heightPerPixel;
+
+    var projection = twgl.m4.ortho(
+        pickLeft, pickRight, pickTop, pickBottom, -1, 1);
+
+    this._drawExcept(Drawable.DRAW_MODE.pick, null, projection);
+
+    var pixels = new Uint8Array(1 * 1 * 4);
+    gl.readPixels(0, 0, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
+
+    return Drawable.color4ubToID(pixels);
+};
diff --git a/src/shaders/sprite.frag b/src/shaders/sprite.frag
index d5db7c5..5cc533b 100644
--- a/src/shaders/sprite.frag
+++ b/src/shaders/sprite.frag
@@ -2,34 +2,39 @@ precision mediump float;
 
 uniform float u_fudge;
 
-#ifdef ENABLE_color
+#ifdef DRAW_MODE_pick
+uniform vec4 u_pickColor;
+#else // DRAW_MODE_pick
+# ifdef ENABLE_color
 uniform float u_color;
-#endif
+# endif // ENABLE_color
+# ifdef ENABLE_brightness
+uniform float u_brightness;
+# endif // ENABLE_brightness
+#endif // DRAW_MODE_pick
+
 #ifdef ENABLE_fisheye
 uniform float u_fisheye;
-#endif
+#endif // ENABLE_fisheye
 #ifdef ENABLE_whirl
 uniform float u_whirl;
-#endif
+#endif // ENABLE_whirl
 #ifdef ENABLE_pixelate
 uniform float u_pixelate;
 uniform vec2 u_skinSize;
-#endif
+#endif // ENABLE_pixelate
 #ifdef ENABLE_mosaic
 uniform float u_mosaic;
-#endif
-#ifdef ENABLE_brightness
-uniform float u_brightness;
-#endif
+#endif // ENABLE_mosaic
 #ifdef ENABLE_ghost
 uniform float u_ghost;
-#endif
+#endif // ENABLE_ghost
 
 uniform sampler2D u_skin;
 
 varying vec2 v_texCoord;
 
-#if defined(ENABLE_color) || defined(ENABLE_brightness)
+#if !defined(DRAW_MODE_pick) && (defined(ENABLE_color) || defined(ENABLE_brightness))
 // Branchless color conversions based on code from:
 // http://www.chilliant.com/rgb2hsv.html by Ian Taylor
 // Based in part on work by Sam Hocevar and Emil Persson
@@ -67,7 +72,7 @@ vec3 convertHSL2RGB(vec3 hsl)
 	float c = (1.0 - abs(2.0 * hsl.z - 1.0)) * hsl.y;
 	return (rgb - 0.5) * c + hsl.z;
 }
-#endif // defined(ENABLE_color) || defined(ENABLE_brightness)
+#endif // !defined(DRAW_MODE_pick) && (defined(ENABLE_color) || defined(ENABLE_brightness))
 
 const vec2 kCenter = vec2(0.5, 0.5);
 
@@ -126,11 +131,20 @@ void main()
 
 	gl_FragColor = texture2D(u_skin, texcoord0);
 
+	#ifdef ENABLE_ghost
+	gl_FragColor.a *= u_ghost;
+	#endif // ENABLE_ghost
+
 	if (gl_FragColor.a == 0.0)
 	{
 		discard;
 	}
 
+	#ifdef DRAW_MODE_pick
+	// switch to u_pickColor only AFTER the alpha test
+	gl_FragColor = u_pickColor;
+	#else // DRAW_MODE_pick
+
 	#if defined(ENABLE_color) || defined(ENABLE_brightness)
 	{
 		vec3 hsl = convertRGB2HSL(gl_FragColor.xyz);
@@ -155,10 +169,8 @@ void main()
 	}
 	#endif // defined(ENABLE_color) || defined(ENABLE_brightness)
 
-	#ifdef ENABLE_ghost
-	gl_FragColor.a *= u_ghost;
-	#endif // ENABLE_ghost
-
 	// WebGL defaults to premultiplied alpha
 	gl_FragColor.rgb *= gl_FragColor.a;
+
+	#endif // DRAW_MODE_pick
 }
