Got it working. The Vector2f class I used has its documentation backwards. I specifically looked which way subtract is when I first tried to do this. It is opposite how they say it. Adjusting accordingly fixed it. Do most people just use JOML? I got this class while I was learning. It’s from MIT.
I backed up and was going to implement a version that just took 4 coordinates and made 4 new coordinates that were 45degrees inward. This is where I found that the subtract is backwards how they say.
I’m going to include my code and some screenshots in case they can help anyone else. Thanks for the thorough help! ;D
My parentheses went ham as I was trying to debug. CBA to delete the superfluous ones now.
//take in mouse coordinates @click and present and make a box
public void drawDragBox3(double mX, double mY, double mX2, double mY2)
{
float lineWidth = 50f;
Vector2f V1 = new Vector2f((float)mX, (float)mY);
Vector2f V2 = new Vector2f((float)mX, (float)mY2);
Vector2f V3 = new Vector2f((float)mX2, (float)mY2);
Vector2f V4 = new Vector2f((float)mX2, (float)mY);
Vector2f L1 = (V2.subtract(V1)).normalize(); //subtract V1 [b]FROM[/b] V2 and then normalize
Vector2f L2 = (V3.subtract(V2)).normalize();
Vector2f L3 = (V4.subtract(V3)).normalize();
Vector2f L4 = (V1.subtract(V4)).normalize();
Vector2f G1 = rotate90Degrees((L1.add(L2)).normalize());
Vector2f G2 = rotate90Degrees((L2.add(L3)).normalize());
Vector2f G3 = rotate90Degrees((L3.add(L4)).normalize());
Vector2f G4 = rotate90Degrees((L4.add(L1)).normalize());
double x1 = 1 / Math.sqrt(((L4.dot(L1))*0.5)+0.5);
double y1 = 1 / Math.sqrt(((L1.dot(L2))*0.5)+0.5);
double x2 = 1 / Math.sqrt(((L1.dot(L2))*0.5)+0.5); //the line theagentd's example was built for
double y2 = 1 / Math.sqrt(((L2.dot(L3))*0.5)+0.5);
double x3 = 1 / Math.sqrt(((L2.dot(L3))*0.5)+0.5);
double y3 = 1 / Math.sqrt(((L3.dot(L4))*0.5)+0.5);
double x4 = 1 / Math.sqrt(((L3.dot(L4))*0.5)+0.5);
double y4 = 1 / Math.sqrt(((L4.dot(L1))*0.5)+0.5);
Vector2f A1 = V1.add(G4.scale((float)(lineWidth/2 * x1)));
Vector2f A2 = V1.subtract((G4.scale((float)(lineWidth/2 * x1)))); //subtract FROM V1
Vector2f A3 = V2.add(G1.scale((float)(lineWidth/2 * y1)));
Vector2f A4 = V2.subtract((G1.scale((float)(lineWidth/2 * y1))));
Vector2f B1 = V2.add(G1.scale((float)(lineWidth/2 * x2)));
Vector2f B2 = V2.subtract((G1.scale((float)(lineWidth/2 * x2))));
Vector2f B3 = V3.add(G2.scale((float)(lineWidth/2 * y2)));
Vector2f B4 = V3.subtract((G2.scale((float)(lineWidth/2 * y2))));
Vector2f C1 = V3.add(G2.scale((float)(lineWidth/2 * x3)));
Vector2f C2 = V3.subtract((G2.scale((float)(lineWidth/2 * x3))));
Vector2f C3 = V4.add(G3.scale((float)(lineWidth/2 * y3)));
Vector2f C4 = V4.subtract((G3.scale((float)(lineWidth/2 * y3))));
Vector2f D1 = V4.add(G3.scale((float)(lineWidth/2 * x4)));
Vector2f D2 = V4.subtract((G3.scale((float)(lineWidth/2 * x4))));
Vector2f D3 = V1.add(G4.scale((float)(lineWidth/2 * y4)));
Vector2f D4 = V1.subtract((G4.scale((float)(lineWidth/2 * y4))));
glDisable(GL_BLEND);
glColor3f(1, 0, 0);
GL11.glLineWidth(lineWidth);
//glEnable(GL_LINE_SMOOTH);
glPolygonMode(GL_FRONT_AND_BACK,GL_FILL);
GL11.glBegin(GL_TRIANGLE_STRIP);
glVertex2f(A1.x, A1.y);
glVertex2f(A4.x, A4.y);
glVertex2f(A3.x, A3.y);
glVertex2f(B4.x, B4.y);
glVertex2f(B3.x, B3.y);
glVertex2f(C4.x, C4.y);
glVertex2f(C3.x, C3.y);
glVertex2f(D4.x, D4.y);
glVertex2f(D3.x, D3.y);
glVertex2f(A4.x, A4.y);
glEnd();
glColor3f(1, 1, 1);
}
Working demo:
Running with glPolygonMode(GL_FRONT_AND_BACK,GL_LINE);
An image I made to help me sort this out: