Below is the method I use for computing normals in a hightfield data set (the getData(x,y) method call gets the height value at the specified x,y coordinates). The main complication is that around the edges, there are viewer values available to do it. Note that I’m using the Java3D vecmath package for the vector math.
/**
* Computes the normal at the grid point (xCoord, yCoord) and assigns it to the first vector in normalCompVecs
* @param xCoord the column of the grid point (strips iterate over these)
* @param yCoord the row of the grid point (one strip each)
* @return the calculated normal vector. Note that this vector may change - make a new one and initialize it to store this normal
*/
private Vector3f computeNormal(int xCoord,int yCoord) {
//uses all of the vectors past the first for computing the normal
Vector3f[] vecs = this.normalCompVecs;
int gran = this.normalGranularity;
float scal = this.normalScaling;
float value = getData(xCoord,yCoord);
if ((vecs.length - 1) == 8) {
int vecsToCross = 1;
int xp1 = xCoord + 1;
int xm1 = xCoord - 1;
int yp1 = yCoord + 1;
int ym1 = yCoord - 1;
//check edges
boolean left = (xCoord == 0);
boolean right = (xCoord >= (getXSize() - gran));
boolean bottom = (yCoord == 0);
boolean top = (yCoord >= (getYSize() - gran));
if (left&&bottom) {
//bottom-left corner
vecs[1].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecs[2].set(gran,gran,scal*(getData(xp1,yp1)-value));
vecs[3].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecsToCross = 3;
} else if (left&&top) {
//upper-left corner
vecs[1].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecs[2].set(gran,-gran,scal*(getData(xp1,ym1)-value));
vecs[3].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecsToCross = 3;
} else if (right&&bottom) {
//bottom-right
vecs[1].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecs[2].set(-gran,gran,scal*(getData(xm1,yp1)-value));
vecs[3].set(-gran,0,scal*(getData(xm1,yCoord)-value));
vecsToCross = 3;
} else if (right&&top) {
//upper-right
vecs[1].set(-gran,0,scal*(getData(xm1,yCoord)-value));
vecs[2].set(-gran,-gran,scal*(getData(xm1,ym1)-value));
vecs[3].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecsToCross = 3;
} else if (left) {
//left edge
vecs[1].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecs[2].set(gran,-gran,scal*(getData(xp1,ym1)-value));
vecs[3].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecs[4].set(gran,gran,scal*(getData(xp1,yp1)-value));
vecs[5].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecsToCross = 5;
} else if (right) {
//right edge
vecs[1].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecs[2].set(-gran,gran,scal*(getData(xm1,yp1)-value));
vecs[3].set(-gran,0,scal*(getData(xm1,yCoord)-value));
vecs[4].set(-gran,-gran,scal*(getData(xm1,ym1)-value));
vecs[5].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecsToCross = 5;
} else if (top) {
//top edge
vecs[1].set(-gran,gran,scal*(getData(xm1,yCoord)-value));
vecs[2].set(-gran,-gran,scal*(getData(xm1,ym1)-value));
vecs[3].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecs[4].set(gran,-gran,scal*(getData(xp1,ym1)-value));
vecs[5].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecsToCross = 5;
} else if (bottom) {
//bottom edge
vecs[1].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecs[2].set(gran,gran,scal*(getData(xp1,yp1)-value));
vecs[3].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecs[4].set(-gran,gran,scal*(getData(xm1,yp1)-value));
vecs[5].set(-gran,0,scal*(getData(xm1,yCoord)-value));
vecsToCross = 5;
} else {
//clockwise from right
vecs[1].set(gran,0,scal*(getData(xp1,yCoord)-value));
vecs[2].set(gran,gran,scal*(getData(xp1,yp1)-value));
vecs[3].set(0,gran,scal*(getData(xCoord,yp1)-value));
vecs[4].set(-gran,gran,scal*(getData(xm1,yp1)-value));
vecs[5].set(-gran,0,scal*(getData(xm1,yCoord)-value));
vecs[6].set(-gran,-gran,scal*(getData(xm1,ym1)-value));
vecs[7].set(0,-gran,scal*(getData(xCoord,ym1)-value));
vecs[8].set(gran,-gran,scal*(getData(xp1,ym1)-value));
vecsToCross = 8;
}
for(int i = 1; i < vecsToCross; ++i)
vecs[i].cross(vecs[i],vecs[i+1]);
if (vecsToCross == 8) {
vecs[8].cross(vecs[8],vecs[1]);
vecsToCross = 9;
}
vecs[0].set(0,0,0);
for(int i = 1; i < (vecsToCross - 1); ++i)
vecs[0].add(vecs[i]);
} else
throw new Exception("Invalid number of vectors for normal calculatuion");
vecs[0].normalize();
return vecs[0];
}