First step:

1. int rectSize = 100;
2. int topCornerX = 250;
3. int topCornerY = 250;
4.  
5. void setup( ) {
6.   size( 500, 500 );
7.   ellipseMode( CENTER );
8.   rectMode( CORNER );
   }
9.  
10. void draw( ) {
11.   background( 255, 0, 0 );
12.  
13.   float rot = PI * 3 / 4 + atan2(topCornerY - mouseY, topCornerX - mouseX);
14.  
15.   pushMatrix( );
16.   translate( topCornerX, topCornerY );
17.   rotate( rot );
18.  
19.   stroke( 0 );
20.   fill( 255 );
21.   rect( 0, 0, rectSize, rectSize );
22.  
23.   fill( 0, 255, 0 );
24.   ellipse( 0, 0, 10, 10 );
25.  
26.   popMatrix( );
27.  
28.   noStroke( );
29.   fill( 0, 0, 255 );
30.   ellipse( mouseX, mouseY, 10, 10 );
31. }

Second step: remove the declaration of rectSize above, use this one just after the declaration of rot:

1.   float rectSize = dist(topCornerY, topCornerX, mouseX, mouseY) * sqrt(2) / 2;

2. 
   
3. void setup( ) {
4.   size( 500, 500 );
5. }
6. void draw( ) {
7.   background( 255 );
8. 
   
9.   int originX = 150;
10.   int originY = 150;
11. 
    
12.   pushMatrix( );
13. 
    
14.   translate( originX, originY );
15.   rotate( radians( 30 ) );  
16. 
    
17.   stroke( 0, 80 );
18.   strokeWeight( 1 );
19.   rectMode( CORNER );
20.   rect( 0, 0, 100, 100 );
21. 
    
22.   float ptAX = screenX( 0, 100 );
23.   float ptAY = screenY( 0, 100 );
24.   float ptBX = screenX( 100, 0 );
25.   float ptBY = screenY( 100, 0 );
26.   popMatrix( );
27. 
    
28.   ellipseMode( CENTER );
29.   float[] perpA = perpPt( originX, originY, ptAX, ptAY, mouseX, mouseY );
30.   ellipse( perpA[0], perpA[1], 10, 10 );
31.   float[] perpB = perpPt( originX, originY, ptBX, ptBY, mouseX, mouseY );
32.   ellipse( perpB[0], perpB[1], 10, 10 );
33. 
    
34.   strokeWeight( 2 );
35.   stroke( 255, 0, 0 );
36.   line( perpA[0], perpA[1], mouseX, mouseY );
37.   line( perpB[0], perpB[1], mouseX, mouseY );
38.   line( originX, originY, perpA[0], perpA[1] );
39.   line( originX, originY, perpB[0], perpB[1] );
40. }
41. 
    
42. 
    
43. float[] perpPt( float x1, float y1, float x2, float y2, float x3, float y3 ) {
44.   float k = ((y2-y1) * (x3-x1) - (x2-x1) * (y3-y1)) / (pow((y2-y1), 2) + pow((x2-x1), 2));
45.   float x4 = x3 - k * (y2-y1);
46.   float y4 = y3 + k * (x2-x1);
47. 
    
48.   float[] fs = {
49.     x4, y4
50.   };
51.   return fs;
52. }

You don't drag any circle, you just start to drag more or less far from the rotation center...

Mmm, not even. When you start dragging, the rotation can be quite different from the previous value, so you see a jump. Then the changes are smoother. If you start dragging close of the location where you ended dragging, there is no jump.

A solution might be to avoid changing rot at a new value, but instead using the delta of movement (between two frames, while dragging) to update it.