With regard to your question, if you want to work with hue, saturation and/or brightness, it's easiest to work in the HSB color mode. You can set the brightness according to the height.

In addition. There were some inefficiencies in your code, like the fact you drag and release everything. I've changed this so you only do it to the one button you have selected. Also I've simplified your click-drag stuff to a more generic isOver() function. Finally I've moved the resize function into the draw loop, so now you can more smoothly resize by pressing the r key and either LEFT mouse (decrease size) or RIGHT mouse (increase size).

Mind you, the code below can be improved more in different areas, but perhaps it will give you an idea how things like those can be done in an alternative manner.

Adapted Code

1. Button[] buttons = new Button[20];
2. int selected;
3.  
4. void setup() {
5.   size(800, 300);
6.   smooth();
7.   colorMode(HSB, 360, 100, 100, 100);
8.   for (int i = 0; i < buttons.length; i++) {
9.     buttons[i]= new Button(random(50,width-50), random(50,height-50), random(20, 80));
10.   }
11. }
12.  
13. void draw() {
14.   background(85);
15.   for (int i = 0; i < buttons.length; i++) {
16.     buttons[i].resize();
17.     buttons[i].display();
18.   }
19. }
20.  
21. void mousePressed() {
22.   for (int i = 0; i < buttons.length; i++) {
23.     if (buttons[i].isOver()) {
24.       selected = i;
25.       break;
26.     }
27.   }
28. }
29.  
30. void mouseDragged() {
31.   if (selected != -1) {
32.     buttons[selected].drag();
33.   }
34. }
35.  
36. void mouseReleased() {
37.   selected = -1;
38. }
39.  
40. class Button {
41.   float x, y;
42.   float rad;
43.   float h, s, b;
44.  
45.   Button(float _x, float _y, float _radius) {
46.     x = _x;
47.     y = _y;
48.     rad = _radius;
49.     setBrightness();
50.   }
51.  
52.   void display() {
53.     noStroke();
54.     fill(h, s, b);
55.     ellipse (x, y, rad, rad);
56.   }
57.  
58.   boolean isOver() {
59.     if (dist(mouseX, mouseY, x, y) <= rad/2) { return true; }
60.     else { return false; }
61.   }
62.  
63.   void resize() {
64.     if (mousePressed && keyPressed && key == 'r' && isOver()) {
65.       if (mouseButton == LEFT) { rad *= 1.1; }
66.       if (mouseButton == RIGHT) { rad *= 0.9; }
67.     }
68.   }
69.  
70.   void setBrightness() {
71.     b = map(y, 0, height, 100, 0);
72.   }
73.  
74.   void drag() {
75.     x = mouseX;
76.     y = mouseY;
77.     setBrightness();
78.   }
79. }

how can i reverse transparency so that transparency is increased as circles are dragged up?

transparency = map(y, 0, height, 0, 100);