Wikidata / Wikipedia API access + "big data" general question

mmodular

hello processing community,

after a long break i have decided to dive into processing again. i wanted to visualize/play around with some statistics and after a few google searches i found that there is a site named "wikidata" where the contents "can be read by humans and machines alike".

i have tried a tool named "autolist" to plot some CSV lists with items and their properties and this is how far ive come: List of all NATO members

the problem is that i cant see the properties of each item. there is a disclaimer that this tool is in development, so im not sure if i'm doing anything wrong or if it is a bug.

So here are the real questions: is there any other, real way to read/export data from wikipedia/wikidata (search for "wikidata" on this forum gave no results)? can someone give me a hint to a general workflow how to get CSV's from there?
are there any other (free) sources to any kind of statistical data (no matter what kind)?

Chrisir

where are you from? What languages do you have?

mmodular

austria, german + english

Chrisir

Hi!

ok, you need to register and get an API-key from them

insert your API-key instead of my XXXXXXXXXXXX below

then you can get weather data here

register:

http://www.wunderground.com/weather/api/d/login.html

data:

http://www.wunderground.com/cgi-bin/findweather/getForecast?query=37.773285,-122.417725&MR=1

documentation:

http://www.wunderground.com/weather/api/d/docs?MR=1

;-)

JSONObject json;

void setup() {

  json = loadJSONObject("http:/" + "/api.wunderground.com/api/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/conditions/q/CA/San_Francisco.json");
  //  println ( json );  // test 

    JSONObject json2 = json.getJSONObject("response");
  //  println ( json2 );   // test 

    // ----
  // #2
  println ( json2.getString("version"));
  // ----
  // # 3
  JSONObject json3 = json2.getJSONObject("features");
  //  println ( json3.getString("conditions"));
  // -----
  JSONObject json4 = json.getJSONObject("current_observation");
  // ------
  JSONObject json5= json4.getJSONObject("image");

  println ( "------------------------------------");
  //println ( json4 );  // test 
  println ( "------------------------------------X");

  println ( json5.getString("title"));
  // ------
  // "display_location"
  JSONObject json6= json4.getJSONObject("display_location");
  println ( json6.getString("full") + " --- " + json6.getString("city") );

  String windString= json4.getString("wind_string");
  println ( "Wind "+windString );

  String temperature_string = json4.getString("temperature_string");
  println ( "Temp "+ temperature_string );
}
//

(mod edit - split url to stop highlighter highlighting)

mmodular

wow, thx... ever worked with wikipedia/wikidata?

Chrisir

not yet...

mmodular

if anyone is interested, here is what i found in "beautiful visualisation" by Julie Steele, Noah Iliinsky

http://books.google.at/books?id=TKh6fdlKwfMC&lpg=PP1&hl=de&pg=PA175#v=twopage&q&f=false

there is an approach but they dont talk about it in the book

Chrisir

can you show the code from the book? Because google doesn't show it.

I made this though.

You have to combine WDQ and Wikidata API

The data you get is incredible complicate and nested...

;-)

//http :// tools.wmflabs.org/wikidata-todo/autolist.html?q=CLAIM%20%5B463%3A7184%5D#
//https :// forum.processing.org/reference/loadJSONObject_.html

/*
get data from WDQ and then from Wikidata API 
 */

/*
WDQ doesn't store names, descriptions, or site names (though you can query if a site link exists). 
 Its purpose is to get you query results, not to be a complete Wikidata mirror. Once you have the 
 numbers from WDQ, you can get such additional data from the Wikidata API (search the page for 
 "action=wbgetentities"); just remember to prefix the numbers with a "Q" :-) 
 --Magnus Manske (talk) 17:17, 12 June 2014 (UTC)
 */

IntList resultListNumbers = new IntList();  
String someDataString = "";
int indexCurrent = 5; 
JSONArray json2;

void setup() {
  size(800, 300);

  JSONObject json = loadJSONObject("http:"+"//wdq.wmflabs.org/api?q=claim[463:7184]");
  json2 = json.getJSONArray ("items") ;

  for (int i = 0; i < min (json2.size(), 20); i++) {
    int item1 = json2.getInt(i);
    resultListNumbers.append(item1);
  } // for 
  someDataString = getSomeDataString(); 
  println ("end of setup()");
} // func 

void draw() {

  background(0);

  String text1="";
  int counter1=0;
  int y=0;
  text ("NATO members", 30, 50);

  for (int i : resultListNumbers) {
    text1 += i+", ";
    if (counter1>10) { 
      text1 += "\n";
      y+=40;
      counter1=0;
    } //if
    counter1++;
  } //for
  //
  text1 += ".";
  y += 40;
  text (text1, 30, 30+y);
  y += 40;
  text (someDataString, 30, 30+y);
  y += 40;
  text ("Please use cursor left and right.", 30, 30+y);
  //
  //noLoop();
}

// -------------------------------------------------------
// Input

void keyPressed() {
  loop(); 
  if (key == CODED) 
  {
    switch (keyCode) {
    case LEFT:
      //
      indexCurrent--;
      if (indexCurrent<=0)
        indexCurrent=0;
      someDataString = getSomeDataString();
      break;
    case RIGHT:
      //
      indexCurrent++;
      if (indexCurrent>=json2.size())
        indexCurrent=json2.size()-1;
      someDataString = getSomeDataString();
      break;
    } // switch
  } //  if (key == CODED) {
  else 
  {
    // not CODED ---------------------- 
    switch (key) {
      //
    } // switch
  } // else not CODED
} // func

// -------------------------------------------------------
// Misc

String getSomeDataString() {
  //   
  int item1 = json2.getInt(indexCurrent);
  someDataString = indexCurrent + ": " + getName (item1) + " is ";
  someDataString += getDescription (item1) + " (" 
    + resultListNumbers.get(indexCurrent) 
    + ").";
  return someDataString;
} // func 

String getDescription (int numberThing ) {
  //println ("------------------------------------");
  println ("facts 1");
  String path1 = "http:"+"//www.wikidata.org/w/api.php?format=json&action=wbgetentities&ids=Q";
  String path2 = numberThing + "&props=descriptions&languages=en";
  println (path1+path2);

  JSONObject json2 = loadJSONObject(path1+path2);
  //  println(json2);
  JSONObject json3 = json2.getJSONObject("entities");
  //println(json3);
  JSONObject json4 = json3.getJSONObject("Q"+numberThing);
  //println(json4);
  JSONObject json5 = json4.getJSONObject("descriptions");
  //println(json5);
  JSONObject json6 = json5.getJSONObject("en");
  //println(json6);
  String valueString = json6.getString("value");
  println(valueString);
  return valueString;
}

String getName (int numberThing ) {
  //println ("------------------------------------");
  println ("facts 2");
  String path1 = "http"+"://www.wikidata.org/w/api.php?format=json&action=wbgetentities&ids=Q";
  String path2 = numberThing     + "&props=labels&sites=enwiki&titles&sitefilter=enwiki&languages=en";
  println (path1+path2);

  JSONObject json2 = loadJSONObject(path1+path2);
  JSONObject json3 = json2.getJSONObject("entities");
  //  println(json3);
  JSONObject json4 = json3.getJSONObject("Q"+numberThing);
  //  println(json4);
  JSONObject json5 = json4.getJSONObject("labels");
  //  println(json5);
  JSONObject json6 = json5.getJSONObject("en");
  //println(json6);
  String valueString = json6.getString("value");
  println(valueString);
  return (valueString);
}
//

Chrisir

and the forum destroys links / urls

you need to "http" + them

Chrisir

see

Dan Shiffman covers the basics of XML: processing.org/tutorials/data/

mmodular

this is amazing, thank you.

i'll try this right now...

Chrisir

hello,

Did you make any progress?

I dream of a sketch where you can type in

• Which states belong to the NATO?

and he identifies "Which states" as a question for states and "belong to the NATO" as a filter. And answers. I haven't understood it fully but as far as I see claim[463:7184] is the key: 463 means country / state and 7184 means NATO. So we have to assign certain keywords with numbers and stitch it together. ;-)

• How many people live in France?

• Which persons were born in Paris 1959?

Best, Chrisir

;-)