'Fix this phong shader for me where to put the lights or vertex position with custom baked normals

My goal is to create a bevel effect for a flat 2d sprite. Where the edges are the start of non transparent pixels.

I've tried this approach Create a bevel effect on non-rectangular shapes in Javascript. And figured I would go with a phong shader.

I copied code from this example http://www.cs.toronto.edu/~jacobson/phong-demo/.

I built normals from custom math along with distance to transparent pixels.

I am not sure I am putting the values correctly but light position and vertex position are uniforms so I can tweak them. Vertex position not sure what it corresponds to but I use texture coordinates, which is pretty close for a quad texture.

I get results like this: enter image description here

But they get pretty off lightining, but seems pretty close. I figure if I set those at the right position this will show bevel highlight on the edges like I wanted.

precision mediump float;

varying vec2 vTextureCoord;
uniform sampler2D uSampler;
uniform vec4 filterArea;

//uniform float transformX;
//uniform float transformY;

const float transformX = 10.0;
const float transformY = 10.0;

bool isalpha(vec2 uv, vec2 offset) {
  vec4 color = texture2D(uSampler, uv - offset);

  return color.a < 0.001;
}

float getdistancetoedge(vec2 uv) {
  float mindistance = 1.0;
  vec2 dir = vec2(1.0 / filterArea) * vec2(transformX, transformY);

  for (int i=0; i <= 20; i++) {
    vec2 vdist = dir * 0.05 * float(i);
    bool alpha = isalpha(uv, vdist);

    if (alpha) {
      mindistance = min(float(i) * 0.05, mindistance);
    }
  }
  return mindistance;
}


const float Kd = 1.0;
const float Ks = 20.0;
const vec3 specularColor = vec3(1.0);
const float shininessVal = 4.0;

uniform vec3 lightPos;
uniform float vertZ;

void main(void) {
  vec4 color = texture2D(uSampler, vTextureCoord);

  vec3 vertPos = vec3(-vTextureCoord, vertZ);

  float dist = getdistancetoedge(vTextureCoord);
  float na = 3.1415 * 0.5 + atan(0.5, dist);

  mat2 rotation = mat2(cos(na), -sin(na), sin(na), cos(na));
  vec2 x_axis = vec2(1.0, 0.0);
  vec2 u_normal = rotation * x_axis;


  vec3 N = vec3(u_normal, 1.0);
  vec3 L = normalize(lightPos - vertPos);
  float specular = 0.0;
  float lambertian = max(dot(N, L), 0.0);

  vec3 R;
  vec3 V;
  float specAngle;
  if (lambertian > 0.0) {
    R = reflect(-L, N);
    V = normalize(-vertPos);
    specAngle = max(dot(R, V), 0.0);
    specular = pow(specAngle, shininessVal);
  }

  color = vec4(Kd * lambertian * color.rgb + Ks * specular * specularColor, 1.0);
  gl_FragColor = vec4(color.rgb * color.a, color.a);
}


Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source