Your description and image make very clear what you're trying to do. But your code was very unintuitive and the variable names incomprehensible. So I just started from scratch to work this out which resulted in something more logical to me, but also completely separate from your code. However it hopefully illustrates the idea/solution, which is as follow...

The key indeed lies in finding the tangent line aka two tangent points. You can do this for a circle quite easily, because the two tangent points are perpendicular to the point's normal which in turn is basically equal to the point's location (this is only true for a circle and a spere by the way).

Note: in the example below I've kept everything in real (non-translated) coordinates to make mouseOver and contains easier to check. This also means I've had to shift a bit back and forth with regard to the center (widht/2, height/2) in relation to accurate vector calculations.

To see it in action, move the mouse over the circle's points. It will display the whole 'above tangent area' (which depends on the normal => sometimes up, down, left, right) and then shows the points that are within this area aka polygon.

Code Example

1. import java.awt.Polygon;
2.  
3. ArrayList <PVector> pointsQuad = new ArrayList <PVector> ();
4. ArrayList <PVector> pointsCircle = new ArrayList <PVector> ();
5. int stepSize = 20;
6. int numCirclePoints = 32;
7. float circleRadius = 100;
8. PVector m = new PVector();
9. Polygon poly;
10.  
11. void setup() {
12.   size(600, 600);
13.   generateQuadPoints();
14.   generateCirclePoints();
15.   poly = new Polygon();
16.   smooth();
17. }
18.  
19. void draw() {
20.   background(255);
21.  
22.   poly.reset();
23.   m.set(mouseX, mouseY, 0);
24.  
25.   for (PVector p : pointsCircle) {
26.     noStroke();
27.     fill(0, 0, 255);
28.     ellipse(p.x, p.y, 5, 5);
29.    
30.     if (p.dist(m)<5) {
31.       PVector pointNormal = new PVector(p.x-width/2, p.y-height/2);
32.       pointNormal.normalize();
33.       PVector tangentOne = new PVector(pointNormal.y, -pointNormal.x);
34.       PVector tangentTwo = new PVector(-pointNormal.y, pointNormal.x);
35.       tangentOne.mult(max(width, height));
36.       tangentTwo.mult(max(width, height));
37.       tangentOne.add(p);
38.       tangentTwo.add(p);
39.       poly.addPoint(int(tangentOne.x), int(tangentOne.y));
40.       poly.addPoint(int(tangentTwo.x), int(tangentTwo.y));
41.       pointNormal.mult(max(width, height));
42.       tangentOne.add(pointNormal);
43.       tangentTwo.add(pointNormal);
44.       poly.addPoint(int(tangentTwo.x), int(tangentTwo.y));
45.       poly.addPoint(int(tangentOne.x), int(tangentOne.y));
46.       stroke(0);
47.       fill(0, 255, 0, 75);
48.       beginShape();
49.       for (int i=0; i<poly.npoints; i++) {
50.         vertex(poly.xpoints[i], poly.ypoints[i]);
51.       }
52.       endShape(CLOSE);
53.     }
54.   }
55.  
56.   for (PVector p : pointsQuad) {
57.     if (poly.contains(int(p.x), int(p.y))) {
58.       stroke(0);
59.       fill(0, 255, 0);
60.       ellipse(p.x, p.y, 10, 10);
61.     } else {
62.       noStroke();
63.       fill(255, 0, 0);
64.       ellipse(p.x, p.y, 5, 5);
65.     }
66.   }
67. }
68.  
69. void generateQuadPoints() {
70.   PVector walker = new PVector();
71.   pointsQuad.add(walker.get());
72.   while (walker.x < width) { walker.x += stepSize; pointsQuad.add(walker.get()); }
73.   while (walker.y < height) { walker.y += stepSize; pointsQuad.add(walker.get()); }
74.   while (walker.x > 0) { walker.x -= stepSize; pointsQuad.add(walker.get()); }
75.   while (walker.y > stepSize) { walker.y -= stepSize; pointsQuad.add(walker.get()); }
76. }
77.  
78. void generateCirclePoints() {
79.   float angle = TWO_PI / numCirclePoints;
80.   for (int i=0; i<numCirclePoints; i++) {
81.     float x = width/2+sin(angle*i)*circleRadius;
82.     float y = height/2+cos(angle*i)*circleRadius;
83.     PVector cp = new PVector(x, y);
84.     pointsCircle.add(cp);
85.   }
86. }