Promblem with unpacking a base64 string

clankill3r

I made this code to compress a multidimensional float array to a compressed base64 string:

import java.util.zip.Deflater; 
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.util.Base64;
import java.lang.*;

// compress

void setup() {

  // 
  float[][] xy_array = new float[][] {{1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}};

  Deflater compresser = new Deflater(Deflater.BEST_COMPRESSION);

  ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
  ByteBuffer byteBuffer = ByteBuffer.allocate(8);

  try {
    for (float[] xy : xy_array) {
      byteBuffer.clear();
      byteBuffer.putFloat(xy[0]);
      byteBuffer.putFloat(xy[1]);
      byteStream.write(byteBuffer.array());
    }
  }
  catch (IOException e) {
    println(e);
  }

  byte[] bytes = byteStream.toByteArray();
  compresser.setInput(bytes);
  compresser.finish();



  byte[] output_buffer = new byte[bytes.length];
  int compressed_data_length = compresser.deflate(output_buffer);
  compresser.end();

  byte[] buffer = new byte[compressed_data_length];
  System.arraycopy(output_buffer, 0, buffer, 0, compressed_data_length);
  String s = Base64.getEncoder().encodeToString(buffer);
  println(s); // eNqzb2BgcGCAYiDhAOIvAOIDQPyAgcERKO4oAMQKQGzAwGA/xNUDAGh4InA=

}

I try to unpack that String. This is work in progress but I can't figure out what leads to the error. It might be even the packer above...

The error I get is:

java.util.zip.DataFormatException: incorrect header check

I hope someone can help.

import java.util.zip.Inflater; 


void setup() {
  String s = "eNqzb2BgcGCAYiDhAOIvAOIDQPyAgcERKO4oAMQKQGzAwGA/xNUDAGh4InA=";

  byte[] input = null;
  try {
    input = s.getBytes("UTF-8");
  }
  catch (Exception e) {
    println(e);
  }

  if (input == null) return;

  Inflater decompresser = new Inflater();
  decompresser.setInput(input);
  byte[] result = new byte[input.length];
  try {
    int resultLength = decompresser.inflate(result);
  }
  catch (Exception e) {
    println(e);
  }
  decompresser.end();

}

koogs

i'd try this one bit at a time

START with a readable string as input data rather than byte encoded floats. this'll make it easier to debug.

then either zip that and save the zip file (you should be able to unzip this using command line tools)

or base64 encode this and save the result (you should also be able to base64 decode this on the command line)

when both those are working then you can zip + base64 encode together

quark

There are 2 major mistakes in the second sketch (decompressor)

1) You don't Base64 decode the string first

2) You set the buffer size equal to the length of the encoded string i.e. the compressed data.

The following sketch provides two methods to encode-the-array and decode-the-string

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Base64;
import java.util.zip.Deflater;
import java.util.zip.Inflater;


void setup() {
  // Test array
  float[][] xy_array = new float[][] {{1, 2}, {2, 3}, {4, 5}, {6, 7}, 
    {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, 
    {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, 
    {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, 
    {4, 5}, {6, 7}, {8, 9}, {10, 11}};
  printArray2D(xy_array);

  String s = shrink(xy_array);
  float[][] decoded_array = enlarge(s);

  printArray2D(decoded_array);
}

public static String shrink(float[][] xy_array) {
  System.out.println("########### ENCODE ARRAY #########");

  // Create byte array from 2D float array
  ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
  ByteBuffer byteBuffer = ByteBuffer.allocate(8);
  try {
    for (float[] xy : xy_array) {
      byteBuffer.clear();
      byteBuffer.putFloat(xy[0]);
      byteBuffer.putFloat(xy[1]);
      byteStream.write(byteBuffer.array());
    }
  }
  catch (IOException e) {
    System.out.println(e);
  }
  byte[] bytes = byteStream.toByteArray();

  // Compress data
  Deflater compresser = new Deflater(Deflater.BEST_COMPRESSION);
  compresser.setInput(bytes);
  compresser.finish();

  // Temporary buffer set to length of uncompressed data
  byte[] temp_buffer = new byte[bytes.length];
  int compressed_data_length = compresser.deflate(temp_buffer);
  compresser.end();

  // Get compressed data as a byte array 
  // (lose unused array elements)
  byte[] compressed_buffer = new byte[compressed_data_length];
  System.arraycopy(temp_buffer, 0, compressed_buffer, 0, compressed_data_length);

  // Base64 encode buffer
  return Base64.getEncoder().encodeToString(compressed_buffer);
}

float[][] enlarge(String s) {
  System.out.println("########### DECODE ARRAY #########");
  // Reverse Base64 decoding
  byte[] buffer = Base64.getDecoder().decode(s);
  // Decompress data
  Inflater decompresser = new Inflater();
  decompresser.setInput(buffer);
  int resultLength = 0;
  // Assume that we achieved 99% compression to avoid over-flowing buffer
  byte[] temp_buffer = new byte[100 * buffer.length];
  try {
    resultLength = decompresser.inflate(temp_buffer);
  }
  catch (Exception e) {
    System.out.println(e);
  }
  decompresser.end();
  // Recreate 2D float array
  DataInputStream byte_stream = new DataInputStream(new ByteArrayInputStream(temp_buffer));
  int nbrFloats = resultLength / 4;
  float[][] array = new float[nbrFloats/2][2];
  try {
    for (int i = 0; i < array.length; i++) {
      array[i][0] = byte_stream.readFloat();
      array[i][1] = byte_stream.readFloat();
    }
  } 
  catch (IOException e) {
    System.out.println(e);
  }  
  return array;
}

void printArray2D(float[][] array) {
  for (float[] array1D : array) {
    System.out.print("{" + array1D[0] + ", " + array1D[1] + ", " + "} ");
  }
  System.out.println();
}

clankill3r

Thanks @quark . I noticed today when being more awake.

I just solved it before getting here. It's really inefficient with creating new arrays of the correct size all the time but it made the code a bit more clear to me to find the problem.

I will look at your code later when I have time (next week...)

import java.util.zip.Deflater; 
import java.util.zip.Inflater; 
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Base64;
import java.lang.*;

class Pack_Unpack {

    public static void main(String[] args) {

        Pack_Unpack pu = new Pack_Unpack();
        //pu.test();
        pu.test2();

    }

    public byte[] compress_bytes(byte[] input) {
        byte[] output = new byte[input.length];
        Deflater compresser = new Deflater();
        compresser.setInput(input);
        compresser.finish();
        int compressedDataLength = compresser.deflate(output);
        compresser.end();

        byte[] right_size_buffer = new byte[compressedDataLength];
        System.arraycopy(output, 0, right_size_buffer, 0, compressedDataLength);

        return right_size_buffer;
    }

    public byte[] decompress_bytes(byte[] input) {
        try {
            Inflater decompresser = new Inflater();
            decompresser.setInput(input);
            byte[] result = new byte[9999];
            int resultLength = decompresser.inflate(result);
            decompresser.end();
            byte[] right_size_buffer = new byte[resultLength];
            System.arraycopy(result, 0, right_size_buffer, 0, resultLength);
            return right_size_buffer;
        }
        catch (java.util.zip.DataFormatException ex) {
            System.out.println("EXCEPTION!");
            System.out.println(ex);
            return null;
        }
    }


    public void test2() {

        float[][] xy_array = new float[][] {{7.2f, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}, {1, 2}, {2, 3}, {4, 5}, {6, 7}, {8, 9}, {10, 11}};   
        byte[] array_as_bytes = float_2d_array_to_byte_array(xy_array);
        byte[] array_as_bytes_compressed = compress_bytes(array_as_bytes);

        byte[] bytes_to_decompress = array_as_bytes_compressed;

        boolean do_base64 = true;
        if (do_base64) {
            // encode to string
            String base64_string = Base64.getEncoder().encodeToString(array_as_bytes_compressed);
            System.out.println(base64_string);
            // Decode from string
            bytes_to_decompress = Base64.getDecoder().decode(base64_string);
        }

        byte[] bytes = decompress_bytes(bytes_to_decompress);

        byte[] four_bytes = new byte[4];
        four_bytes[0] = bytes[0];
        four_bytes[1] = bytes[1];
        four_bytes[2] = bytes[2];
        four_bytes[3] = bytes[3];
        //float f = ByteBuffer.wrap(four_bytes).order(ByteOrder.LITTLE_ENDIAN).getFloat();
        float f = ByteBuffer.wrap(four_bytes).getFloat();


        System.out.println(f);

        System.out.println("DONE");


    }


    public byte[] float_2d_array_to_byte_array(float[][] in) {

        try {
            // catch(java.io.UnsupportedEncodingException ex) {
                ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
                ByteBuffer byteBuffer = ByteBuffer.allocate(8);

                for (float[] xy : in) {
                    byteBuffer.clear();
                    byteBuffer.putFloat(xy[0]);
                    byteBuffer.putFloat(xy[1]);
                    byteStream.write(byteBuffer.array());
                }

                byte[] bytes = byteStream.toByteArray();

                return bytes;
            }
            catch (Exception ex) {
                System.out.println("EXCEPTION!");
                System.out.println(ex);
                return null;
            }
        }


    }