How can I visualise tweets on a map in Processing?

NatashaP

I'm trying to use the twitter API to search for a keyword and get the location of that particular tweet to then visualise onto a map.

I've successfully created my map using unfolding maps and tilemill, I'm just struggling with the twitter part. Using the twitter 4J library, I've tried the following code but I'm not able to get the location as coordinates. Also, once I've got the coordinates I dont know how to go about visualising them on my map.

import twitter4j.conf.*;
import twitter4j.api.*;
import twitter4j.*;

import java.util.List;
import java.util.Iterator;

ConfigurationBuilder   cb;
Query query;
Twitter twitter;

ArrayList<String> twittersList;
Timer             time;

//Number twitters per search
int numberSearch = 10;

PFont font;
int   fontSize = 14;

void setup() {
  size(670, 650);
  background(0);
  smooth(4);

  //FONT
  font = createFont("NexaLight-16.vlw", fontSize, true);
 textFont(font, fontSize);

  //You can only search once every 1 minute
  time = new Timer(70000); //1 min with 10 secs

  //Acreditacion
  cb = new ConfigurationBuilder();
  cb.setOAuthConsumerKey("**********");
  cb.setOAuthConsumerSecret("***********");
  cb.setOAuthAccessToken("**************");
  cb.setOAuthAccessTokenSecret("*************");

  //Make the twitter object and prepare the query
  twitter = new TwitterFactory(cb.build()).getInstance();

  //SEARCH
  twittersList = queryTwitter(numberSearch);
}

void draw() {
  background(50);

  //draw twitters
  drawTwitters(twittersList);


  if (time.isDone()) {
    twittersList = queryTwitter(numberSearch);
    time.reset();
   }

  text(time.getCurrentTime(), 20, 30);

  time.update();
}

 void drawTwitters(ArrayList<String> tw) {
   Iterator<String> it = tw.iterator();
  int i = 0;

  while (it.hasNext ()) {
      String twitt = it.next();
    fill(150);
    text(i + 1, 27, 60 + i*(fontSize)*4 + fontSize);
    fill(220);
    text(twitt, 50, 60 + i*(fontSize)*4, 600,  fontSize*4);
    i++;
  }
}

 double lat = 63.894947; 
double lon = -22.054045;
double res = 10;
String resUnit = "mi";

ArrayList<String> queryTwitter(int nSearch) {
  ArrayList<String> twitt = new ArrayList<String>();

  query = new Query("#happy");
  query.setCount(nSearch);
  try {
    QueryResult result = twitter.search(new Query().geoCode(new GeoLocation(lat, lon),res, resUnit));
    List<Status> tweets = result.getTweets();
    println("New Tweet : ");
    for (Status tw : tweets) {
      String msg = tw.getText();
       String usr = tw.getUser().getScreenName();
      String twStr = "@"+usr+": "+msg;
      println(twStr);
      twitt.add(twStr);
    }


  }
  catch (TwitterException te) {
    println("Couldn't connect: " + te);
  }

  return twitt;
    }

I'm quite new to processing, so please be patient with me as I don't even know if I'm going about this the right way.

Any help would be gratefully appreciated!!

koogs

you have two things to do. the good thing is that you don't have to do them both at the same time. so

a) pick some random points and get those drawing on the map.

b) get the coords from the tweet and just println them, ignoring the map. try tw.getGeoLocation().

http://twitter4j.org/javadoc/twitter4j/Status.html#getGeoLocation--

http://twitter4j.org/javadoc/twitter4j/GeoLocation.html

then just add a and b together...

NatashaP

Thanks for your reply. I've now managed to get the tweet locations using tw.getGeoLocation(); as you suggested .... thank you!

I now want to plot these coordinates onto my map but don't know how to go about this. Also my twitter search is only coming back with about 3 tweets (depending on my search word), is this because of how I've set it up? I used part of an example which included a timer, but don't know if it's restricting the number of tweets it returns?

import twitter4j.conf.*;
import twitter4j.api.*;
import twitter4j.*;

import java.util.List;
import java.util.Iterator;

ConfigurationBuilder   cb;
Query query;
Twitter twitter;

ArrayList<String> twittersList;
Timer             time;

//Number twitters per search
int numberSearch = 50;

PFont font;
int   fontSize = 18;

void setup() {
  size(670, 650);
  background(0);
 smooth(4);

  //FONT
  font = createFont("NexaLight-16.vlw", fontSize, true);
  textFont(font, fontSize);

  //You can only search once every 1 minute
  time = new Timer(70000); //1 min with 10 secs

  //Acreditacion
  cb = new ConfigurationBuilder();
  cb.setOAuthConsumerKey("**********");
   cb.setOAuthConsumerSecret("*********");
  cb.setOAuthAccessToken("**********");
  cb.setOAuthAccessTokenSecret("******");

  //Make the twitter object and prepare the query
  twitter = new TwitterFactory(cb.build()).getInstance();

  //SEARCH
  twittersList = queryTwitter(numberSearch);
}

void draw() {
  background(50);

  //draw twitters
  drawTwitters(twittersList);


  if (time.isDone()) {
    twittersList = queryTwitter(numberSearch);
    time.reset();
  }

  text(time.getCurrentTime(), 20, 30);

  time.update();
}

void drawTwitters(ArrayList<String> tw) {
  Iterator<String> it = tw.iterator();
  int i = 0;

  while (it.hasNext ()) {
    String twitt = it.next();
    fill(150);
    text(i + 1, 27, 60 + i*(fontSize)*4 + fontSize);
    fill(220);
    text(twitt, 50, 60 + i*(fontSize)*4, 600,  fontSize*4);
    i++;
  }
}

 double lat = 63.879158;  
double lon = -22.444955;
double res = 10;
String resUnit = "mi";

ArrayList<String> queryTwitter(int nSearch) {
  ArrayList<String> twitt = new ArrayList<String>();


  try {
    QueryResult result = twitter.search(new Query("#geothermal").geoCode(new     GeoLocation(lat,lon), res, resUnit));

    List<Status> tweets = result.getTweets();
    println("New Tweet : ");
    for (Status tw : tweets) {

         double latitude = tw.getGeoLocation().getLatitude();
         double longitude = tw.getGeoLocation().getLongitude();




      String msg = tw.getText();
      String usr = tw.getUser().getScreenName();
      String twStr = "@"+usr+": " +latitude + longitude + "-" +msg ;
      println(twStr);
      twitt.add(twStr);
         text( "Longtitude: "+longitude, 20,15);
         text( "Latitude:  "+latitude, 20,15);
    }


  }
  catch (TwitterException te) {
   println("Couldn't connect: " + te);
  }

  return twitt;
}

koogs

Be careful of lines 94 and 95. If the tweet doesn't contain a location you'll get a null pointer exception.

The timer is restricting how often the tweet search is done by the look of it and I don't think it's a problem. The search itself is restricting the location of the tweet to 10 miles from the provided coords, which might be restrictive, but I guess that's deliberate on your part.

Will try and dig out my Twitter keys and run it myself.

koogs

yeah, change it to 20 miles and you'll get 5.

(50 and 100 miles return a lot of results with no lat/long in the tweet, triggering the NPE problem i mentioned above)

NatashaP

Thank so much for you help.

Basically, I only want tweets from Iceland as that's where my project is based, so does this mean I am only going to be able to get a maximum of 5 returns with locations?

Also I don't really understand the null pointer exception error. I noticed originally I was getting this, so played around with the numbers until I stopped getting the error, but would ideally like to know how to get around this.

koogs

just check to see if the data is null before trying to use it

double latitude, longitude;
if (tw != null && tw.getGeoLocation() != null) {
  // safe to use lat long
  latitude = tw.getGeoLocation()..getLatitude();
  longitude = tw.getGeoLocation().getLongitude();
}

koogs

btw, you can do the same search via twitter.com

#geothermal geocode:63.879158,-22.444955,20mi

gives me a bunch of results. 5 from 2017 and the next most recent after that is last november. i guess the blue lagoon closes for christmas 8)

NatashaP

Ah ok that makes sense, thanks :)

I've got one more problem if you don't mind helping...sorry I know I'm asking a lot haha but I am very new to this. I've never done any form of coding before, I'm a postgrad architecture student and have just been introduced to this for a data-mining exercise.

So i've managed to combine the two files .... one being the twitter search and the other my map Ive styled. The problem I'm now having is adding the marker of the tweet locations to the map. I've done a draft exercise from this tutorial: http://unfoldingmaps.org/tutorials/markers-simple

Rather than manually inputting the location points, I'm looking for a way to link it to the location points of the tweets the twitter sketch has returned. So far I've got the following code, but its creating an ambiguous error for the Location type. After doing a bit of research, I gather this is due to both library's containing the Location class, but how do I solve this? Or is there a better way to add the markers?

Map tab :

import de.fhpotsdam.unfolding.*;
import de.fhpotsdam.unfolding.core.*;
import de.fhpotsdam.unfolding.data.*;
import de.fhpotsdam.unfolding.events.*;
import de.fhpotsdam.unfolding.geo.*;
import de.fhpotsdam.unfolding.interactions.*;
import de.fhpotsdam.unfolding.mapdisplay.*;
import de.fhpotsdam.unfolding.mapdisplay.shaders.*;
import de.fhpotsdam.unfolding.marker.*;
import de.fhpotsdam.unfolding.providers.*;
import de.fhpotsdam.unfolding.texture.*;
import de.fhpotsdam.unfolding.tiles.*;
import de.fhpotsdam.unfolding.ui.*;
import de.fhpotsdam.unfolding.utils.*;
import de.fhpotsdam.utils.*;

UnfoldingMap currentmap;
SimplePointMarker twitterMarker;


String[] paths;

void setup() {
  fullScreen();
  smooth(10);
  setup_twitter();

  processData();

  MapUtils.createDefaultEventDispatcher(this, currentmap);
  Location tweetLocation = new Location(Latitude, Longitude);

  // Create point markers for locations
  SimplePointMarker tweetMarker = new SimplePointMarker(tweetLocation);

  // Add markers to the map
  map.addMarkers(tweetMarker);
}

void processData() {
  String mbTilesString = sketchPath("data/healthcentrelocations.mbtiles");
  currentmap = new UnfoldingMap(this, new MBTilesMapProvider(mbTilesString));
}

void draw() {
  currentmap.draw();
}

Twitter tab:

import twitter4j.conf.*;
import twitter4j.api.*;
import twitter4j.*;
import java.util.List;
import java.util.Iterator;


ConfigurationBuilder   cb;
Query query;
Twitter twitter;

ArrayList<String> twittersList;


//Number twitters per search
int numberSearch = 50;

void setup_twitter () {

  //Acreditacion
 cb = new ConfigurationBuilder();
  cb.setOAuthConsumerKey("*****");
  cb.setOAuthConsumerSecret("*****");
  cb.setOAuthAccessToken("*****");
  cb.setOAuthAccessTokenSecret("*****");

  //Make the twitter object and prepare the query
  twitter = new TwitterFactory(cb.build()).getInstance();

  //SEARCH
  twittersList = queryTwitter(numberSearch);
}

double lat = 63.879158;  
double lon = -22.444955;
double res = 10;
String resUnit = "mi";

ArrayList<String> queryTwitter(int nSearch) {
  ArrayList<String> twitt = new ArrayList<String>();


  try {
    QueryResult result = twitter.search(new Query("#geothermal").geoCode(new GeoLocation(lat, lon), res, resUnit));

    List<Status> tweets = result.getTweets();
    println("New Tweet : ");
    for (Status tw : tweets) {


      double latitude, longitude;
      if (tw != null && tw.getGeoLocation() != null) {
        // safe to use lat long
        latitude = tw.getGeoLocation().getLatitude();
        longitude = tw.getGeoLocation().getLongitude();


        //String msg = tw.getText();
        String usr = tw.getUser().getScreenName();
        String twStr = "@"+usr+": " +latitude + longitude ;
        println(twStr);
        twitt.add(twStr);
        //  text( "Longtitude: "+longitude, 20,15);
        // text( "Latitude:  "+latitude, 20,15);
      }
    }
  }
  catch (TwitterException te) {
    println("Couldn't connect: " + te);
  }

  return twitt;
}

koogs

ambiguous error for the Location type

use the Fully Qualified Name for the class so it knows which one to use

de.fhpotsdam.unfolding.geo.Location tweetLocation = new de.fhpotsdam.unfolding.geo.Location(Latitude, Longitude);

(this wouldn't normally be an issue because the code is in separate files. but processing concats all the .pde source files together before compiling...)