Pixels

In Processing we can access the pixels of an image or those that are displayed on the screen. When accessing them we can pull out the color values and modify it for display.

Here is a quick example that we’ll review:

// Don't forget WRITE_EXTERNAL_STORAGE

PImage theImage;

void setup() {
  //size(400,400);
  theImage = loadImage("dottie.jpg");
}

void draw() {
  loadPixels();
  theImage.loadPixels();

  // Loop through the x
  for (int x = 0; x < theImage.width && x < width; x++ ) {
    // For each x, loop through the y
    for (int y = 0; y < theImage.height && y < height; y++ ) {
      
      // Calculate the array pixel location for the image
      int imageIndex = x + y*theImage.width;
      // Calculate the array pixel location for the screen
      int screenIndex = x + y*width;
      
      // Get the red, green and blue values from image
      float r = red (theImage.pixels[imageIndex]);
      float g = green (theImage.pixels[imageIndex]);
      float b = blue (theImage.pixels[imageIndex]);
      
      // The closer the pixel is to the center, the lower the distance
      float distance = dist(x, y, theImage.width/2, theImage.height/2);

      float maxDistance = sqrt(sq(theImage.width) + sq(theImage.height));
        
      float distanceRatio = distance/maxDistance;

      // We want closer pixels to be brighter
      r = r*(1-distanceRatio);
      g = g*(1-distanceRatio);
      b = b*(1-distanceRatio);

      // We also want sepia
      r = r + 99;
      g = g + 66;
      b = b + 33;

      // Constrain RGB to between 0-255
      r = constrain(r, 0, 255);
      g = constrain(g, 0, 255);
      b = constrain(b, 0, 255);
        
      // Make a new color out of the new RGB values and set the pixels of the sketch
      color c = color(r, g, b);
      pixels[screenIndex] = c;
    }
  }

  // Call updatePixels to tell processing to draw the pixel values to the screen
  updatePixels();
  
  filter(BLUR, 2);
}

void mousePressed() {
 // Save output to the SD card
 save(android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/output.tif"); 
}

Checkout the filter and blend references as well.

Select an Existing Image

We can create an Intent that allows the user to select an existing image using the built-in capabilities and display that image in our sketch.

Intent choosePictureIntent = new Intent(Intent.ACTION_PICK,␣ android.provider.MediaStore.Images.Media.EXTERNAL_CONTENT_URI);

Here is an example:

final int CHOOSE_PICTURE = 0;

String imageFilePath;
PImage selectedImage;

void setup() {
  
}

void draw() {
  if (selectedImage != null) {
    image(selectedImage,0,0,width,height); 
  }
}

void mousePressed() {
  android.content.Intent choosePictureIntent = new android.content.Intent(android.content.Intent.ACTION_PICK, android.provider.MediaStore.Images.Media.EXTERNAL_CONTENT_URI);
  startActivityForResult(choosePictureIntent, CHOOSE_PICTURE);
}

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    
    android.net.Uri imageFileUri = intent.getData();
    // This Uri is a "content://" style intent which Processing can't use directly.  We need to query the MediaStore to get the file path
    //println(imageFileUri.toString());
    // content://media/external/images/media/9397
    
    String[] columns = { android.provider.MediaStore.Images.Media.DATA }; // The file path
    android.database.Cursor cursor = managedQuery(imageFileUri, columns, null, null, null);
    if (cursor.moveToFirst()) { 
      imageFilePath = cursor.getString(cursor.getColumnIndexOrThrow(android.provider.MediaStore.Images.Media.DATA));
      println(imageFilePath);
      selectedImage = loadImage(imageFilePath);
    }
  }
}

Image Capture with an Intent

As with most media capture capabilities available on Android, we can use an Intent to leverage the the built-in application. To use the built-in camera application, we create an Intent like this:

// Path to where we want the file and what to call it
String imageFilePath = android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/myfavoritepicture.jpg";
// Create a File object out of that
File imageFile = new File(imageFilePath);
// Create a Uri out of that
android.net.Uri imageFileUri = android.net.Uri.fromFile(imageFile);

// Create the Intent that triggers the camera
android.content.Intent i = new android.content.Intent(android.provider.MediaStore.ACTION_IMAGE_CAPTURE);
// Tell the camera application where we want the resulting image saved
i.putExtra(android.provider.MediaStore.EXTRA_OUTPUT, imageFileUri);
// Start the Camera
startActivityForResult(i, CAMERA_RESULT);

To get the resulting image, we need to implement an onActivityResult method in our sketch:

PImage cameraImage;

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    // We know the location via the imageFilePath String so load it into a standard Processing PImage
    cameraImage = loadImage(imageFilePath);
  }

Here is a full example:

final int CAMERA_RESULT = 0;

String imageFilePath;
File imageFile;
android.net.Uri imageFileUri;

PImage cameraImage;

void setup() {
  imageFilePath = android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/myfavoritepicture.jpg";
  imageFile = new File(imageFilePath);
  imageFileUri = android.net.Uri.fromFile(imageFile);
}

void draw() {
  if (cameraImage != null) {
    image(cameraImage,0,0,width,height); 
  }
}

void mousePressed() {
  android.content.Intent i = new android.content.Intent(android.provider.MediaStore.ACTION_IMAGE_CAPTURE);
  i.putExtra(android.provider.MediaStore.EXTRA_OUTPUT, imageFileUri);
  startActivityForResult(i, CAMERA_RESULT);
}

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    cameraImage = loadImage(imageFilePath);
  }
}

AudioRecorder

Just a quick example:

class AudioRecorder
{
  int frequency = 8000;
  boolean keepGoing = false;
  android.media.AudioRecord audioRecord;
  int bufferSize;
  public short[] buffer;
  PApplet sketch;

  java.lang.reflect.Method updateBufferMethod;
  
  public AudioRecorder(PApplet _sketch) {
    sketch = _sketch;
    
    bufferSize = android.media.AudioRecord.getMinBufferSize(
      frequency, 
      android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, 
      android.media.AudioFormat.ENCODING_PCM_16BIT);
      
    audioRecord = new android.media.AudioRecord(
      android.media.MediaRecorder.AudioSource.MIC,
      frequency,
      android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO,
      android.media.AudioFormat.ENCODING_PCM_16BIT,
      bufferSize);
      
    buffer = new short[bufferSize];

    // check to see if the host applet implements
    // public void updateBufferMethod(short[] buffer)
    try {
      updateBufferMethod = sketch.getClass().getMethod("updateBuffer", new Class[] { AudioRecorder.class });
    } catch (Exception e) {
      // just ignore
    }

  }

  public void stop() {
    keepGoing = false;
  }

  public void record() {
    keepGoing = true;				
    audioRecord.startRecording();
    new Thread(new Runnable() {
      public void run() {
        while (keepGoing) {
           audioRecord.read(buffer, 0, bufferSize);
           
           // Send to Processing sketch
           if (updateBufferMethod != null) {
             try {
               updateBufferMethod.invoke(sketch, new Object[] { this });
              } catch (Exception e) {
                System.err.println("Disabling updateBuffer() because of an error.");
                e.printStackTrace();
                updateBufferMethod = null;
              }           
           }
        }
        audioRecord.stop();
      }
    }).start();
  }

  short[] getBuffer() {
    return buffer;
  }
}

Main Sketch


AudioRecorder ar;
boolean recording = false;
public short[] buffer;

void setup() {
  ar = new AudioRecorder(this);
}

void draw() {
  
}

void mousePressed() {
  if (recording == false) {
    println("Going to record");
    ar.record();
    recording = true;
  } else {
    println("Stop recording");
    ar.stop();
    
    buffer = ar.getBuffer();
    for (int i = 0; i < buffer.length; i++) {
      println(buffer[i]);
    }
    
  }  
}

/*
void updateBuffer(AudioRecorder recorder) {
  println("update buffer");
  for (int i = 0; i < recorder.buffer.length; i++) {
      println(recorder.buffer[i]);
    }
}
*/