To rotate point pos1 about point center by a radians

1. // needed for temporary storage
2. float nx, ny;
3. // Makes sense to calculate the sine and cosine just once
4. float sinAngle = sin(a);
5. float cosAngle = cos(a);
6. // Calculate new points
7. nx = pos1.x * cosAngle - pos1.y * sinAngle;
8. ny = pos1.x * sinAngle + pos1.y * cosAngle;
9. pos1.x = nx;
10. pos1.y = ny;
11. // Continue for other points

You need nx and ny because because you must not change x and y until BOTH new values are calculated.

I tried but something is still wrong.

The formula I gave you was rotation about the origin i.e. (0,0) but you want rotation about another point 'center' this works

1. PVector pos1, pos2, pos3, pos4;
2. PVector p1, p2, p3, p4;
3. PVector center;
4. 
   
5. void setup() {
6.   size(400, 400);
7.   p1 = new PVector(50, 50);
8.   p2 = new PVector(width-50, 100);
9.   p3 = new PVector(width-50, 200);
10.   p4 = new PVector(50, 300);
11.   // p1 --- p2
12.   // |      |
13.   // |      |
14.   // p4 --- p3
15.   center = new PVector(width/2, height/2);
16. }
17. 
    
18. void draw() {
19.   background(150);
20.   // create a set of vectors relative to vector'center'
21.   pos1 = PVector.sub(p1, center);
22.   pos2 = PVector.sub(p2, center);
23.   pos3 = PVector.sub(p3, center);
24.   pos4 = PVector.sub(p4, center);
25. 
    
26.   // rotation from center to mouse
27.   float a = atan2(mouseY - center.y, mouseX - center.x);
28.   // quad(pos1.x, pos1.y, pos2.x, pos2.y, pos3.x, pos3.y, pos4.x, pos4.y);
29.   // needed for temporary storage
30.   float nx, ny;
31.   // Makes sense to calculate the sine and cosine just once
32.   float sinAngle = sin(a);
33.   float cosAngle = cos(a);
34.   // Calculate new points
35.   nx = pos1.x * cosAngle - pos1.y * sinAngle;
36.   ny = pos1.x * sinAngle + pos1.y * cosAngle;
37.   pos1.x = nx;
38.   pos1.y = ny;
39.   // Continue for other points
40.   nx = pos2.x * cosAngle - pos2.y * sinAngle;
41.   ny = pos2.x * sinAngle + pos2.y * cosAngle;
42.   pos2.x = nx;
43.   pos2.y = ny;
44.   nx = pos3.x * cosAngle - pos3.y * sinAngle;
45.   ny = pos3.x * sinAngle + pos3.y * cosAngle;
46.   pos3.x = nx;
47.   pos3.y = ny;
48.   nx = pos4.x * cosAngle - pos4.y * sinAngle;
49.   ny = pos4.x * sinAngle + pos4.y * cosAngle;
50.   pos4.x = nx;
51.   pos4.y = ny;
52. 
    
53.   pos1.add(center);
54.   pos2.add(center);
55.   pos3.add(center);
56.   pos4.add(center);
57. 
    
58.   stroke(0);
59.   strokeWeight(1.5);
60.   fill(255);
61.   quad(pos1.x, pos1.y, pos2.x, pos2.y, pos3.x, pos3.y, pos4.x, pos4.y);
62. 
    
63.   // to rotation should be in this direction:
64.   stroke(255, 0, 0);
65.   pushMatrix();
66.   translate(center.x, center.y);
67.   rotate(a);
68.   line(0, 0, 100, 0);
69.   popMatrix();
70. }

thanks a million! :)