Ah right - I needed to bring the region check for the line segments into the if-else bit - nearly there now (added a quick optimisation too), just one final problem:
Code:

Point [] p;
void setup(){
  size(400, 400, P3D);
  p = new Point[4];
  p[0] = new Point(200, 100);
  p[1] = new Point(200,200);
  p[2] = new Point(80,180);
  p[3] = new Point(50,100);
}
void draw(){
  background(255);
  quad(p[0].x, p[0].y, p[1].x, p[1].y, p[2].x, p[2].y, p[3].x, p[3].y);
  ellipse(mouseX, mouseY, 50, 50);
  if(circlePolyCollision(mouseX, mouseY, 25, p)){
    fill(0);
  } 
  else {
    fill(255);
  }
}
boolean circlePolyCollision(float xc, float yc, float r, Point [] p){
  Point c = new Point(xc,yc);
  for(int i = 1; i < p.length+1; i++){
    Point a = p[i-1];
    Point b = p[(i%p.length)];
    Line segment = new Line(a, b);
    Line to_circle = new Line(a, c);
    // vertex region check
    float len = Line.dot(segment, segment);
    float dp = Line.dot(to_circle, segment);
    if(dp < 0){
	// a is the closest vertex
	if(proximity(a.x, a.y, c.x, c.y, r)) return true;
    } else if(dp > len){
	// b is the closest vertex
	if(proximity(b.x, b.y, c.x, c.y, r)) return true;
    } else if(dp >=0 && dp <=len){
	// segment region check - check distance to line
	float np = (to_circle.vx*-segment.lx)+(to_circle.vy*-segment.ly);
	line(c.x, c.y, c.x+np*segment.lx, c.y+np*segment.ly);
	float vx = np*segment.lx;
	float vy = np*segment.ly;
	float d = vx*vx+vy*vy;
	if((r*r)-d >= 0){
	  return true;
	}
    }

  }
  return false;
}
boolean proximity(float x0, float y0, float x1, float y1, float len){
  return (x1-x0)*(x1-x0)+(y1-y0)*(y1-y0) <= len*len;
}
static class Point{
  float x,y;
  Point(float x, float y){
    this.x = x;
    this.y = y;
  }
}
static class Line{
  Point a, b;
  float vx,vy,dx,dy,len,rx,ry,lx,ly;
  Line(Point a, Point b){
    this.a = a;
    this.b = b;
    updateLine();
  }
  void updateLine(){
    vx = b.x-a.x;
    vy = b.y-a.y;
    len = sqrt(vx*vx+vy*vy);
    // normalized unti-sized components
    if (len>0) {
	dx = vx/len;
	dy = vy/len;
    } 
    else {
	dx = 0;
	dy = 0;
    }
    rx = -dy;
    ry = dx;
    lx = dy;
    ly = -dx;
  }
  static float dot(Line a, Line b){
    return a.vx*b.vx+a.vy*b.vy;
  }
}
 

How do I register a positive when the circle is inside the polygon?

Okay, I could test the center of the circle as to whether it is inside the poly - but thats adding a huge hurdle that I automatically have to hobble over to rule out if the circle is inside. I only get to skip that check if I get a collision! I'm pretty sure there must be a vector way of doing it. In fact this nifty page on polygon collision refers to voronoi regions inside the polygon. If that's true I can do away with the trig. I'm just a bit stumped at the moment - but you do have a valid point. I guess if I don't find an answer I can always rely on using a bounding box check against the circle and poly to cut overhead. But I'm damned sure there's another way...


On a related note - I found something on a similar page as your strip of code from here

I used to use the angle measuring method but I'm wondering if all of those atan2() calls are a bit expensive. It also lists doing a line-intersection check. Basically if you get an odd number casting out from your line then you have a point inside a polygon. Here's a sketch re-mashing that C stuff into something sensible (or not when you look at the if statement):
Code:

Point [] p;
void setup(){
  size(400, 400, P3D);
  p = new Point[4];
  p[0] = new Point(200, 100);
  p[1] = new Point(200,200);
  p[2] = new Point(80,180);
  p[3] = new Point(50,100);
}
void draw(){
  background(255);
  quad(p[0].x, p[0].y, p[1].x, p[1].y, p[2].x, p[2].y, p[3].x, p[3].y);
  if(insidePolygon(mouseX, mouseY, p)) fill(0); else fill(255);
}
boolean insidePolygon(float x, float y, Point [] p){
  int i, j, c = 0;
  for (i = 0, j = p.length-1; i < p.length; j = i++) {
    if ((((p[i].y <= y) && (y < p[j].y)) ||
	((p[j].y <= y) && (y < p[i].y))) &&
	(x < (p[j].x - p[i].x) * (y - p[i].y) / (p[j].y - p[i].y) + p[i].x))
	c = (c+1)%2;
  }
  return c==1;
}
static class Point{
  float x,y;
  Point(float x, float y){
    this.x = x;
    this.y = y;
  }
}
 

This also seems a bit long in the tooth too, but it works all the same. Might be faster, might not - needs a test I guess.

here's what i have so far.

P3D was giving the strange results with complex polys not insidePolygon() which works very good. anyway i leave the link to bourke:
http://local.wasp.uwa.edu.au/~pbourke/geometry/insidepoly/

Code:

Point[] polygon;
Circle circle; // x,y,r

void setup ()
{
    size( 200, 200, P3D ); // just for the per vertex coloring
    generateNewLines();
    circle = new Circle(0,0,0);
    frameRate( 24 );
}

void draw ()
{
    background(Colors.white);

    if (mousePressed ) {
        circle = new Circle(mouseX,mouseY,5);
    }

    fill( Colors.grey );
    stroke( Colors.black );

    beginShape(POLYGON);
    int i = 0;
    boolean inside = insidePolygon(circle.x, circle.y, polygon);
    while (true)
    {
        boolean intersect = inside || lineIntersectsCircle( polygon[i], polygon[(i+1)%polygon.length], circle);
        stroke( intersect ? Colors.blue : Colors.black );
        fill( intersect ? Colors.blue : Colors.grey );
        vertex( polygon[i].x, polygon[i].y );
        i ++;
        if ( i % polygon.length == 0 ) break;
    }
    endShape(CLOSE);

    circle.drawCircle();
}

void keyReleased ()
{
    generateNewLines();
}

void generateNewLines()
{
    int count = ((int)random(3,10));
    polygon = new Point[count];
    for( int i = 0; i<polygon.length; i++ )
    {
        polygon[i] = new Point(random(width),random(height));
    }     
}

boolean lineIntersectsCircle ( Point p1, Point p2, Circle c )
{
    float ldx = p2.x-p1.x;
    float ldy = p2.y-p1.y;
    float lk = ldy / ldx;
    float ld = p1.y - p1.x * lk;
    /*
   // draw parallel line
     float cd = c.y - c.x * lk; // parallel line thru circle center
     stroke( Colors.blue );
     float cyy = (c.x+ldx) * lk + cd;
     line (c.x,c.y,c.x+ldx,cyy);
     */
    float cd2 = c.y - c.x * (-1.0/lk); // parallel line thru circle center

    /*
   // draw perpendicular line
     stroke( Colors.green );
     float cyyy =  (c.x+ldx) * (-1.0/lk) + cd2;
     line (c.x,c.y,c.x+ldx,cyyy);
     */

    // how this is solved:
    // left side: c.y = c.x * (-1.0/lk) + cd2
    // right side: p1.y = p1.x * lk + ld
    // where c.y = p1.y = y and c.x = p1.x = x
    // x * (-1.0/lk) + cd2 = x * lk + ld
    // x * ((-1.0/lk)-lk) = ld - cd2
    // x = (ld - cd2) / ((-1.0/lk)-lk)

    float xx = (ld - cd2) / ((-1.0/lk)-lk);
    float yy = xx * lk + ld;

    /*
   // draw connection
     stroke( Colors.red );
     line( xx,yy,c.x,c.y );
     */

    if ( !(xx >= min( p1.x, p2.x ) & xx <= max( p1.x, p2.x ) & yy >= min( p1.y, p2.y ) & yy <= max( p1.y, p2.y )) ) return false;

    float rr = sqrt((c.x-xx)*(c.x-xx)+(c.y-yy)*(c.y-yy));

    return c.rad >= rr;
}

boolean insidePolygon ( float x, float y, Point [] p)
{
    // aaron steed
    // http://processing.org/discourse/yabb_beta/YaBB.cgi?board=Programs;action=display;num=1189178826;start=4#4
    int i, j, c = 0; 
    for (i = 0, j = p.length-1; i < p.length; j = i++) { 
        if ((((p[i].y <= y) && (y < p[j].y)) || 
            ((p[j].y <= y) && (y < p[i].y))) && 
            (x < (p[j].x - p[i].x) * (y - p[i].y) / (p[j].y - p[i].y) + p[i].x)) 
            c = (c+1)%2; 
    } 
    return c==1; 
}

class Circle
{
    float x,y,rad, dia;
    
    Circle ( float _x, float _y, float _rad )
    {
        x = _x;
        y = _y;
        rad = _rad;
        dia = 2.0 * _rad;
    }

    void drawCircle ()
    {
        fill( Colors.white );
        stroke( Colors.red );
        ellipse( x, y, dia, dia );
    }
}

static class Point { 
    float x,y; 
    Point(float x, float y){ 
        this.x = x; 
        this.y = y; 
    } 
}

class Colors
{
    final static int red   = 0xFFFF0000;
    final static int green = 0xFF00FF00; 
    final static int blue  = 0xFF0000FF;   
    final static int black = 0xFF000000;    
    final static int grey  = 0xFFAAAAAA;     
    final static int white = 0xFFFFFFFF; 
}