I'm reading through Ben Fry's 'Visualizing Data' and I'm writing the scatter plot map program in Chapter 6 (If you haven't read it, it's essentially about drawing GPS coordinates). I've gotten the base program working, but now I'm trying to edit it to display more information.

My basic question is how do I read my input file, and only bring a single row into memory at once, as opposed to the whole file. The entire file I've got is too big to bring into memory all at once. I've got a large dataset I want to try to draw, but the way the program is currently written, it brings the entire input into memory at once, and then draws it. I'd like to loop through each row in my input, draw that, dump it from memory, draw the next row, and so on.

Does anyone have any tips?

1. // The first line in the input file, in this case, zips2.csv contains the bounding box for the drawing area
   // The first number is the count of the total number of rows, then it's MinX, MaxX, MinY, MaxY
   
   // Column numbers in the data file
   static final int X = 0;
   static final int Y = 1;
   
   int totalCount; // Total number of places
   Place[] places;
   int placeCount; // Numbers of places loaded
   
   // MIN/MAX boundary of all points
   float minX, maxX;
   float minY, maxY;
   
   // Border of where the map should be drawn on screen
   float mapX1, mapY1;
   float mapX2, mapY2;
   
   public void setup() {
     size(1280, 720, P2D);
   
     mapX1 = 3;
     mapX2 = width - mapX1;
     mapY1 = 2;
     mapY2 = height - mapY1;
    
     readData();
   }
   
   void readData() {
     String[] lines = loadStrings("lat_lons.csv");
     parseInfo(lines[0]); // Read the header line
    
     places = new Place[totalCount];
     for (int i = 1; i < lines.length; i++) {
       places[placeCount] = parsePlace(lines[i]);
       placeCount++;
     }
   }
   
   void parseInfo(String line) {
     String infoString = line.substring(2); // Remove the #
     String[] infoPieces = split(infoString, ',');
     totalCount = int(infoPieces[0]);
     minX = float(infoPieces[1]);
     maxX = float(infoPieces[2]);
     minY = float(infoPieces[3]);
     maxY = float(infoPieces[4]);
   }
   
   Place parsePlace(String line) {
     String pieces[] = split(line, ',');
    
     float x = float(pieces[X]);
     float y = float(pieces[Y]);
    
     return new Place(x, y);
   }
   
   public void draw() {
     background(255);
     for (int i = 0; i < placeCount; i++) {
       places[i].draw();
     }
   }
   
   float TX(float x) {
     return map(x, minX, maxX, mapX1, mapX2);
   }
   
   float TY(float y) {
     return map(y, minY, maxY, mapY2, mapY1);
   }
2. class Place {
     float x;
     float y;
    
     public Place(float x, float y) {
       this.x = x;
       this.y = y;
     }
    
     void draw() {
       int xx = (int) TX(x);
       int yy = (int) TY(y);
       set(xx, yy, #000000);
     }
   }