Small JSON Retrieve of a digital library (meta data only)

Chrisir

hello all,

as some of you know, the German Digital library has published a API for their meta data. You need to get an API-key from them which takes a few days and then you can retrieve information about titles of books etc.

See http://www.heise.de/newsticker/meldung/Deutsche-Digitale-Bibliothek-laesst-per-API-auf-Metadaten-zugreifen-2041752.html

http://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001

Anyway, I made a JSON query for one title. Only one title.

The query is somewhat compley since the main JSONobject contains other JSONobjects and even a JSONarray.

You need to get your own API-Key and write it into the code.

I failed to load the image of the insitution the book belongs to. The institution is shown in red.

All rights are by the Deutsche-Digitale-Bibliothek.

Greetings, Chrisir

//
// URL of DDB server with dataset ID and requested method
final String ddb_url = "https://api.deutsche-digitale-bibliothek.de/items/4XJ4PI7Q4LAG5IWBTV2J57RLUM6WBRDL/view";  // public key for one book 
final String ddb_key = "";   // API-key - get your own
//
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;

// 

void setup () {
  size (1200, 798);
}
//
void draw () {
  background(12);
  showData ();
  println("-----------------------------");
  println ("Done");
  println("-----------------------------");
  noLoop();
}


void showData () {

  // get JSON data via query parameter authentication

  // build query expression  
  String queryJsonURL = "";
  try {
    // remember: use URL encoded Strings online -> URLEncoder.encode(s, enc)
    queryJsonURL = ddb_url + "?oauth_consumer_key=" + URLEncoder.encode(ddb_key, "UTF-8");
    println (queryJsonURL);
  }
  catch (Exception e0) {  // Encoding Exception 
    // failed
    println ("Error e0 = " +e0 + " internal error number is 4242.");
    exit();
  } 

  println("---------------------------------------------------");
  println("");
  //

  JSONObject values = null;

  try {
    values = loadJSONObject(queryJsonURL);
  }
  catch (Exception e1) {  //  Exception 
    // failed
    println ("Error e1 = " +e1 + " internal error number is 8080.");
    exit();
  } 

  if (values == null) {
    println ("values is null, sketch stops ");
    println ("   You probabaly must get a API-key from http://www.deutsche-digitale-bibliothek.de  ");
    println ("   see http://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001 ");
  } // if 
  else 
  {
    // this evaluates the JSONObject 
    evaluate (values);
  } // else
} // func 

// -------------------------------------------------------------

void evaluate (JSONObject values) {
  println("values");
  println(values);
  JSONObject itemValue1 = values.getJSONObject("item");

  println("--------------------------------------------------");
  println("itemValue1:");
  println(itemValue1);

  // the title (here are some more values that are not shown here)
  textSize (22);
  text (itemValue1.getString ("title"), 20, 27 );

  JSONObject itemValue2 = itemValue1.getJSONObject("fields");
  println("-----------------------------");
  println("itemValue2");
  println(itemValue2);

  // ----------------------------------------------------

  println("--------------------------------------------");
  println("for-loop ");
  int y=90;
  textSize (18);
  JSONArray itemValue3 = itemValue2.getJSONArray("field");

  for (int i = 0; i < itemValue3.size(); i++) {

    JSONObject infoEntity = itemValue3.getJSONObject(i); 

    String name  = infoEntity.getString("name");
    String value = infoEntity.getString("value");
    String id    = infoEntity.getString("@id");

    text(name + 
      ": \n       " + 
      value + 
      " (@id: " + id+")", 
    30, y );

    y+=80; // next infoEntity

    // println(name + ", " + value + ", " + id);
  } // for
  //  
  JSONObject institutionValue1 = itemValue1.getJSONObject("institution");

  println("-----------------------------");
  println("-----------------------------");
  println("institution ");
  println(institutionValue1 );

  String institutionName =   institutionValue1.getString ("name");

  // this doesn't work: 
  // Load image from a web server
  String whatToLoad =  findLinkInURL ( institutionValue1.getString ("logo") );
  try {
    // whatToLoad =  URLEncoder.encode( whatToLoad, "UTF-8");
  }
  catch (Exception eImage) {
    // do nothing
  }
  PImage institutionImage =  loadImage ( whatToLoad, "jpg" );
  if (institutionImage!=null)
    image (institutionImage, width-300, 150);

  // this works : 
  fill(244, 3, 3);
  text ("Institution", width-300, 90);
  text (institutionValue1.getString ("name"), width-300, 110);
  text (institutionValue1.getString ("url"), width-300, 130);
  //
} // func 

// 

String findLinkInURL (String url) {

  // expects a hyperlink like <a href="link"> yet text </a> and gives you the "link" 

  int start = url.indexOf("<a href=\"") + 9;
  int stop  = url.indexOf("\">");

  String buffer = url.substring(start, stop);
  println ( buffer );
  return buffer;
} // func 
// 

PhiLho

What does the content of whatToLoad looks like?

Chrisir

It's a link to an Image

It's the return value of the func findLinkInURL

quote:

 ---(the func) expects a hyperlink like <a href="link"> yet text </a> and gives you the "link" 

Chrisir

ok, I 'll check and insert a println there...

maybe I overlooked a " or so ...

Thank you...

PhiLho

"It's a link to an Image"
Yes, but I wanted to see this link... Is it absolute? Relative? http? https? ftp? Etc.

Chrisir

PhiLho, thank you, this is it:

http://content.deutsche-digitale-bibliothek.de/binaries/institution-institutionen/20120720/edit/00005860/BSB_Logo_2c_300dpi rgb_srgb.jpg

I didn't read the docu on images from the German Library, maybe they say something about loading binaries

there's a space ' ' in it but it works if I copy the entire line in my browser...

Chrisir

oh, now it works, it was indeed caused by the space ' ' in the link whatToLoad which I had to replace by %20

Thank you!!!

see line 137 below

full code:

//
// URL of DDB server with dataset ID and requested method
final String ddb_url = "https://api.deutsche-digitale-bibliothek.de/items/4XJ4PI7Q4LAG5IWBTV2J57RLUM6WBRDL/view";  // public key for one book 
final String ddb_key = "gMVFu1Xr2nkT86NxYLWAg97972E5D5ncy707fBp8y7c1vk1iPM81384353151171";   // API-key - get your own
//

// see https://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001



import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;

// 

void setup () {
  size (1200, 798);
}
//
void draw () {
  background(12);
  showData ();
  println("-----------------------------");
  println ("Done");
  println("-----------------------------");
  noLoop();
}


void showData () {

  // get JSON data via query parameter authentication

  // build query expression  
  String queryJsonURL = "";
  try {
    // remember: use URL encoded Strings online -> URLEncoder.encode(s, enc)
    queryJsonURL = ddb_url + "?oauth_consumer_key=" + URLEncoder.encode(ddb_key, "UTF-8");
    println (queryJsonURL);
  }
  catch (Exception e0) {  // Encoding Exception 
    // failed
    println ("Error e0 = " +e0 + " internal error number is 4242.");
    exit();
  } 

  println("---------------------------------------------------");
  println("");
  //

  JSONObject values = null;

  try {
    values = loadJSONObject(queryJsonURL);
  }
  catch (Exception e1) {  //  Exception 
    // failed
    println ("Error e1 = " +e1 + " internal error number is 8080.");
    exit();
  } 

  if (values == null) {
    println ("values is null, sketch stops ");
    println ("   You probabaly must get a API-key from http://www.deutsche-digitale-bibliothek.de  ");
    println ("   see http://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001 ");
  } // if 
  else 
  {
    // this evaluates the JSONObject 
    evaluate (values);
  } // else
} // func 

// -------------------------------------------------------------

void evaluate (JSONObject values) {
  println("values");
  println(values);
  JSONObject itemValue1 = values.getJSONObject("item");

  println("--------------------------------------------------");
  println("itemValue1:");
  println(itemValue1);

  // the title (here are some more values that are not shown here)
  textSize (22);
  text (itemValue1.getString ("title"), 20, 27 );

  JSONObject itemValue2 = itemValue1.getJSONObject("fields");
  println("-----------------------------");
  println("itemValue2");
  println(itemValue2);

  // ----------------------------------------------------

  println("--------------------------------------------");
  println("for-loop ");
  int y=90;
  textSize (18);
  JSONArray itemValue3 = itemValue2.getJSONArray("field");

  for (int i = 0; i < itemValue3.size(); i++) {

    JSONObject infoEntity = itemValue3.getJSONObject(i); 

    String name  = infoEntity.getString("name");
    String value = infoEntity.getString("value");
    String id    = infoEntity.getString("@id");

    text(name + 
      ": \n       " + 
      value + 
      " (@id: " + id+")", 
    30, y );

    y+=80; // next infoEntity

    // println(name + ", " + value + ", " + id);
  } // for
  //  
  JSONObject institutionValue1 = itemValue1.getJSONObject("institution");

  println("-----------------------------");
  println("-----------------------------");
  println("institution ");
  println(institutionValue1 );

  String institutionName =   institutionValue1.getString ("name");

  // Load institution image from a web server
  String whatToLoad =  findLinkInURL ( institutionValue1.getString ("logo") );
  whatToLoad = whatToLoad.replace ( " ", "%20" ); 
  PImage institutionImage =  loadImage ( whatToLoad, "gif" );

  println (whatToLoad);
  if (institutionImage!=null)
    image (institutionImage, width-300, 150);
  else 
    println ("loading image failed. >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");

  // other institution data : 
  fill(244, 3, 3);
  text ("Institution", width-300, 90);
  text (institutionValue1.getString ("name"), width-300, 110);
  text (institutionValue1.getString ("url"), width-300, 130);
  //
} // func 

// 

String findLinkInURL (String url) {

  // expects a hyperlink like <a href="link"> some text </a> and gives you the "link" 
  // (you might have to replace ( " ", "%20" ) afterwards) 

  int start = url.indexOf("<a href=\"") + 9;
  int stop  = url.indexOf("\">");

  String buffer = url.substring(start, stop);
  println ( buffer );
  return buffer;
} // func 
// 

Chrisir

I now made a complete new program that can search the database

API-key - get your own, sorry

it comes with two states

[edited]

//
// URL of DDB server with dataset ID and requested method
final String ddb_url = "https://api.deutsche-digitale-bibliothek.de/search";  // public key for one book 
final String ddb_key = "";   // API-key - get your own
//

// see https://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001

import java.net.URLEncoder;
import controlP5.*;

// states 
final int stateInput = 0;
final int stateResults = 1;

int state =  stateInput;

// lib ControlP5 
ControlP5 cp5;

Textfield myTextfield; 
String textValue = "";  // the Input (search term) 
Textarea myTextarea;    // the text area 
String myTextareaText;  // the result 

String headLine = ""; 


void setup () {
  size (1200, 798);
  if (ddb_key.equals(""))
  {
    println ("   ddb_key is empty - You probabaly must get a API-key from http://www.deutsche-digitale-bibliothek.de  ");
    println ("   see http://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001 ");
    exit();
  }


  PFont font = createFont("arial", 20);

  cp5 = new ControlP5(this);

  // text field 
  myTextfield  = cp5.addTextfield("input")
    .setPosition(20, 100)
      .setSize(200, 40)
        .setFont(font)
          .setFocus(true)
            .setColor(color(255, 0, 0))
              ;

  // text area 
  myTextarea = cp5.addTextarea("txt")
    .setPosition(100, 100)
      .setSize(700, 600)
        .setFont(createFont("arial", 12))
          .setLineHeight(14)
            .setColor(color(128))
              .setColorBackground(color(255, 100))
                .setColorForeground(color(255, 100));
  myTextarea.hide();


  textFont(font);
} // func 
//
void draw () {
  // MAIN 
  background(12);

  // depending on the state 
  switch (state) {

  case stateInput:
    // wait for Input 
    // at the end textfield for input is hidden and query is sende and textarea is shown   
    break;

  case stateResults:
    if (!textValue.equals("")) {     
      if (keyPressed && key==' ') {
        myTextarea.scroll((float)mouseX/(float)width);
      }
      if (keyPressed && key=='l') {
        myTextarea.setLineHeight(mouseY);
      }
      if (keyPressed && (key==RETURN||key==ENTER)) {
        // go back to search / first state -----------------
        myTextarea.hide();
        myTextfield.show(); 
        myTextfield.setFocus(true);
        headLine="";
        textValue="";
        myTextareaText="";
        state = stateInput; // state ----------------------
      }
      text(headLine, 20, 20);
    } // if 
    break;

  default:
    // Error 
    println ("Error 260");
    exit();
    break;
  } // switch
} // func 

void getData () {

  // get JSON data via query parameter authentication

  // build query expression  
  String queryJsonURL = "";
  try {
    // remember: use URL encoded Strings online -> URLEncoder.encode(s, enc)
    // Space is replaced by + 
    textValue = textValue.replace(' ', '+');
    // build the query String with the Input textValue
    queryJsonURL = ddb_url + "?oauth_consumer_key=" 
      + URLEncoder.encode(ddb_key, "UTF-8") 
      + "&query=" 
        + textValue ;  // + "&query=Justus+Jonas" ; 
    println (queryJsonURL);
  }
  catch (Exception e0) {  // Encoding Exception 
    // failed
    println ("Error e0 = " +e0 + " internal error number is 4242.");
    exit();
  } 

  println("---------------------------------------------------");
  println("");
  //

  JSONObject values = null;

  try {
    values = loadJSONObject(queryJsonURL);
  }
  catch (Exception e1) {  //  Exception 
    // failed
    println ("Error e1 = " +e1 + " internal error number is 8080.");
    exit();
  } 

  if (values == null) {
    println ("values is null, sketch stops ");
    println ("   You probabaly must get a API-key from http://www.deutsche-digitale-bibliothek.de  ");
    println ("   see http://www.deutsche-digitale-bibliothek.de/content/news/2013-11-04-001 ");
  } // if 
  else 
  {
    // this evaluates the JSONObject 
    evaluate (values);
  } // else
} // func 

// -------------------------------------------------------------

void evaluate (JSONObject values) {
  println("values");
  println(values);
  JSONArray itemValue1 = values.getJSONArray ("results");

  println("--------------------------------------------------");
  println("itemValue1:");
  println(itemValue1);
  println("--------------------------------------------");
  println("for-loop ");

  JSONArray theDocs = null;
  for (int i = 0; i < itemValue1.size(); i++) {
    JSONObject infoEntity = itemValue1.getJSONObject(i); 
    String name  = infoEntity.getString("name");
    int numberOfDocs  = infoEntity.getInt("numberOfDocs");
    theDocs = infoEntity.getJSONArray("docs");
    headLine = textValue 
      + ": " 
      + name 
      + "; number of docs:  " 
      + numberOfDocs 
      + ". Hit Return to search again.";
  } // for

  println ( theDocs.size() + " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
  for (int i = 0; i < theDocs.size(); i++) {
    JSONObject infoEntity = theDocs.getJSONObject(i); 
    String title  = infoEntity.getString("title");
    String subtitle  = "";
    try {
      subtitle  = infoEntity.getString("subtitle");
    }
    catch (Exception e99) {
      // do nothing
    }
    myTextareaText=  myTextareaText  +  "\n\n" +  title + "\n(" + subtitle+")";
  } // for
  //
} // func 

// 

//String findLinkInURL (String url) {
//
//  // expects a hyperlink like <a href="link"> some text </a> and gives you the "link" 
//  // (you might have to replace ( " ", "%20" ) afterwards) 
//
//  int start = url.indexOf("<a href=\"") + 9;
//  int stop  = url.indexOf("\">");
//
//  String buffer = url.substring(start, stop);
//  println ( buffer );
//  return buffer;
//} // func 
// 


// =========================================
// for Textarea 
//void changeWidth(int theValue) {
//  myTextarea.setWidth(theValue);
//}
//
//void changeHeight(int theValue) {
//  myTextarea.setHeight(theValue);
//}
// =========================================

// for text input field 

void controlEvent(ControlEvent theEvent) {
  if (theEvent.isAssignableFrom(Textfield.class)) {
    println("controlEvent: accessing a string from controller '"
      +theEvent.getName()+"': "
      +theEvent.getStringValue()
      );
    // prepare next state  ----------------------------------------------
    textValue=theEvent.getStringValue();
    // hide text field 
    cp5.get(Textfield.class, "input").hide();
    getData ();  // this gets JSON Data
    myTextarea.scroll(0);
    myTextarea.show(); // show the text area 
    myTextarea.setText(myTextareaText);  // and fill it with text 
    println ("Hit Return to search again");
    // next state  ---------------------------------------------------
    state = stateResults;
  }
}

public void input(String theText) {
  // automatically receives results from controller input
  println("a textfield event for controller 'input' : "+theText);
}

// =========================================

Chrisir

I made a new version with search, you get a full srollable menu with the results (thanks to ControlP5!) and you can click a result and get the detail informaton of that book.

Any interested person please drop me a line..