We can create an Intent that allows the user to select an existing image using the built-in capabilities and display that image in our sketch.

Intent choosePictureIntent = new Intent(Intent.ACTION_PICK,␣ android.provider.MediaStore.Images.Media.EXTERNAL_CONTENT_URI);

Here is an example:

final int CHOOSE_PICTURE = 0;

String imageFilePath;
PImage selectedImage;

void setup() {
  
}

void draw() {
  if (selectedImage != null) {
    image(selectedImage,0,0,width,height); 
  }
}

void mousePressed() {
  android.content.Intent choosePictureIntent = new android.content.Intent(android.content.Intent.ACTION_PICK, android.provider.MediaStore.Images.Media.EXTERNAL_CONTENT_URI);
  startActivityForResult(choosePictureIntent, CHOOSE_PICTURE);
}

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    
    android.net.Uri imageFileUri = intent.getData();
    // This Uri is a "content://" style intent which Processing can't use directly.  We need to query the MediaStore to get the file path
    //println(imageFileUri.toString());
    // content://media/external/images/media/9397
    
    String[] columns = { android.provider.MediaStore.Images.Media.DATA }; // The file path
    android.database.Cursor cursor = managedQuery(imageFileUri, columns, null, null, null);
    if (cursor.moveToFirst()) { 
      imageFilePath = cursor.getString(cursor.getColumnIndexOrThrow(android.provider.MediaStore.Images.Media.DATA));
      println(imageFilePath);
      selectedImage = loadImage(imageFilePath);
    }
  }
}

As with most media capture capabilities available on Android, we can use an Intent to leverage the the built-in application. To use the built-in camera application, we create an Intent like this:

// Path to where we want the file and what to call it
String imageFilePath = android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/myfavoritepicture.jpg";
// Create a File object out of that
File imageFile = new File(imageFilePath);
// Create a Uri out of that
android.net.Uri imageFileUri = android.net.Uri.fromFile(imageFile);

// Create the Intent that triggers the camera
android.content.Intent i = new android.content.Intent(android.provider.MediaStore.ACTION_IMAGE_CAPTURE);
// Tell the camera application where we want the resulting image saved
i.putExtra(android.provider.MediaStore.EXTRA_OUTPUT, imageFileUri);
// Start the Camera
startActivityForResult(i, CAMERA_RESULT);

To get the resulting image, we need to implement an onActivityResult method in our sketch:

PImage cameraImage;

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    // We know the location via the imageFilePath String so load it into a standard Processing PImage
    cameraImage = loadImage(imageFilePath);
  }

Here is a full example:

final int CAMERA_RESULT = 0;

String imageFilePath;
File imageFile;
android.net.Uri imageFileUri;

PImage cameraImage;

void setup() {
  imageFilePath = android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/myfavoritepicture.jpg";
  imageFile = new File(imageFilePath);
  imageFileUri = android.net.Uri.fromFile(imageFile);
}

void draw() {
  if (cameraImage != null) {
    image(cameraImage,0,0,width,height); 
  }
}

void mousePressed() {
  android.content.Intent i = new android.content.Intent(android.provider.MediaStore.ACTION_IMAGE_CAPTURE);
  i.putExtra(android.provider.MediaStore.EXTRA_OUTPUT, imageFileUri);
  startActivityForResult(i, CAMERA_RESULT);
}

protected void onActivityResult(int requestCode, int resultCode, android.content.Intent intent) { 
  super.onActivityResult(requestCode, resultCode, intent);
  if (resultCode == RESULT_OK) {
    cameraImage = loadImage(imageFilePath);
  }
}

Just a quick example:

class AudioRecorder
{
  int frequency = 8000;
  boolean keepGoing = false;
  android.media.AudioRecord audioRecord;
  int bufferSize;
  public short[] buffer;
  PApplet sketch;

  java.lang.reflect.Method updateBufferMethod;
  
  public AudioRecorder(PApplet _sketch) {
    sketch = _sketch;
    
    bufferSize = android.media.AudioRecord.getMinBufferSize(
      frequency, 
      android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, 
      android.media.AudioFormat.ENCODING_PCM_16BIT);
      
    audioRecord = new android.media.AudioRecord(
      android.media.MediaRecorder.AudioSource.MIC,
      frequency,
      android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO,
      android.media.AudioFormat.ENCODING_PCM_16BIT,
      bufferSize);
      
    buffer = new short[bufferSize];

    // check to see if the host applet implements
    // public void updateBufferMethod(short[] buffer)
    try {
      updateBufferMethod = sketch.getClass().getMethod("updateBuffer", new Class[] { AudioRecorder.class });
    } catch (Exception e) {
      // just ignore
    }

  }

  public void stop() {
    keepGoing = false;
  }

  public void record() {
    keepGoing = true;				
    audioRecord.startRecording();
    new Thread(new Runnable() {
      public void run() {
        while (keepGoing) {
           audioRecord.read(buffer, 0, bufferSize);
           
           // Send to Processing sketch
           if (updateBufferMethod != null) {
             try {
               updateBufferMethod.invoke(sketch, new Object[] { this });
              } catch (Exception e) {
                System.err.println("Disabling updateBuffer() because of an error.");
                e.printStackTrace();
                updateBufferMethod = null;
              }           
           }
        }
        audioRecord.stop();
      }
    }).start();
  }

  short[] getBuffer() {
    return buffer;
  }
}

Main Sketch

AudioRecorder ar;
boolean recording = false;
public short[] buffer;

void setup() {
  ar = new AudioRecorder(this);
}

void draw() {
  
}

void mousePressed() {
  if (recording == false) {
    println("Going to record");
    ar.record();
    recording = true;
  } else {
    println("Stop recording");
    ar.stop();
    
    buffer = ar.getBuffer();
    for (int i = 0; i < buffer.length; i++) {
      println(buffer[i]);
    }
    
  }  
}

/*
void updateBuffer(AudioRecorder recorder) {
  println("update buffer");
  for (int i = 0; i < recorder.buffer.length; i++) {
      println(recorder.buffer[i]);
    }
}
*/

What is the best camera?

Why is the iPhone the top camera on Flickr? Most Popular Cameras in the Flickr Community

Mobile phones have cameras built in that rival the best point and shoot cameras out there. Some even have optical zoom, built-in flashes and so on.

Not only capture but … (capture, storage, editing, viewing, sharing)

Impossible before, possible now:

Polaroid Android

Say, Can You Make Phone Calls on That Camera?

HDR 35 Fantastic HDR Pictures

Instagram

What about?

Cameras, the new lighter

The rise of the camera-phone
Everywhere you go these days, there are people with camera-phones – many of us record, document, and upload the minutae of our lives. But, ultimately, should we be doing it just because we can?

Why are we compelled to document everything? Are we missing anything in the process?

What about privacy? Activism/Protest? Surveillance?
Illicit photos?

Vancouver Riot Identify Suspects
Arab Spring
Remove Location Information on Photos

Android has a built-in class called AudioTrack that can be used to play or synthesize audio.

Here is a code snippet:

final int SAMPLE_RATE = 11025;
int minSize = 0; 
short[] buffer = {8130,15752,22389,27625,31134,32695,32210,29711,25354,19410,12253, 4329,-3865,-11818,-19032,-25055,-29511,-32121,-32722,-31276,-27874, -22728,-16160,-8582,-466};

boolean playing = false;

android.media.AudioTrack audioTrack;
    
void setup() {
  minSize = android.media.AudioTrack.getMinBufferSize(SAMPLE_RATE, android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, android.media.AudioFormat.ENCODING_PCM_16BIT);
  
  audioTrack = new android.media.AudioTrack(android.media.AudioManager.STREAM_MUSIC, 
    SAMPLE_RATE, android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, 
    android.media.AudioFormat.ENCODING_PCM_16BIT, minSize,
    android.media.AudioTrack.MODE_STREAM);
    
  audioTrack.play();
  
}

void draw() {

}

void mousePressed() {
  if (!playing) {
    playing = true; 

    while (true) { 
      audioTrack.write(buffer, 0, buffer.length);
    }
  }
}

If you run this, it should generate a static tone but you will notice a couple of problems. The first is that the app becomes unresponsive and the second is that Android wants you to quit it.

This is because we are running the sound playback in an infinite loop and it is tying up the “UI thread”.

A Thread is a concurrently running piece of code that your program can create to handle tasks that would otherwise occupy the main part of the program. In a sense, it is running these other portions of your program in the background, simultaneously. Of course, this is a very simplified explanation, see Java’s Thread documentation and the Wikipedia Thread article.

What we really want to do is run the audio playback on a separate thread.

AudioSynth Class:

class AudioSynth 
{
  final int SAMPLE_RATE = 11025;
  int minSize = 0; 
  short[] buffer = {8130,15752,22389,27625,31134,32695,32210,29711,25354,19410,12253, 4329,-3865,-11818,-19032,-25055,-29511,-32121,-32722,-31276,-27874, -22728,-16160,-8582,-466};
  android.media.AudioTrack audioTrack;
        
  boolean keepGoing = false;


  public AudioSynth() {

    minSize = android.media.AudioTrack.getMinBufferSize(SAMPLE_RATE, android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, android.media.AudioFormat.ENCODING_PCM_16BIT);
  
    audioTrack = new android.media.AudioTrack(android.media.AudioManager.STREAM_MUSIC, 
      SAMPLE_RATE, android.media.AudioFormat.CHANNEL_CONFIGURATION_MONO, 
      android.media.AudioFormat.ENCODING_PCM_16BIT, minSize,
      android.media.AudioTrack.MODE_STREAM);
  }
  
  public void stop() {
    keepGoing = false;
  }
  
  public void play() {
    keepGoing = true;
    audioTrack.play();				
  
    new Thread(new Runnable() {
      public void run() {
        while (keepGoing) {
           audioTrack.write(buffer, 0, buffer.length);
        }
        audioTrack.stop();
      }
    }
    ).start();
  }
  
  void setBuffer(short[] newbuffer) {
   buffer = newbuffer; 
  }
}

Main Processing Sketch:

boolean playing = false;
AudioSynth synth;
    
void setup() {
  synth = new AudioSynth();      
}

void draw() {

}

void mousePressed() {
  if (!playing) {
    playing = true; 
    synth.play();
  } else {
    playing = false;
    synth.stop(); 
  }
}

Now that’s all well and good but what if we want to dynamically generate the sound to play?

Here is a really annoying version:

boolean playing = false;
AudioSynth synth;
    
void setup() {
  synth = new AudioSynth();  
  synth.play();  
}

void draw() {

}

void mousePressed() {
  short[] buffer = {(short)(mouseX*100),(short)(mouseY*100)};
  synth.setBuffer(buffer);
}

In many cases, people will use math to generate samples. Here is a sine wave (at mouseX frequency):

boolean playing = false;
AudioSynth synth;
    
void setup() {
  synth = new AudioSynth();  
  synth.play();  
}

void draw() {

}

void mousePressed() {
  short[] buffer = new short[synth.minSize];
  float angle = 0;
  float angular_frequency = (float)(2*Math.PI) * mouseX / synth.SAMPLE_RATE;
    // 2 x PI is the angle in radians of 360 degrees
    // multiply that by the mouseX/frequency of samples, you get the angle in radians of a specific frequency (C, 440Hz)
    // http://www.motionscript.com/mastering-expressions/simulation-basics-1.html
    
  for (int i = 0; i < buffer.length; i++)
  {
    buffer[i] = (short)(Short.MAX_VALUE * ((float) Math.sin(angle)));
    angle += angular_frequency;
  }  
  
  synth.setBuffer(buffer);
}

Audio Playback using an Intent

The easiest and most straightforward way to do many things on Android is to leverage an existing piece of software on the device by using an intent. An intent is a core component of Android that is described in the documentation as a “description of an action to be performed.” In practice, intents are used to trigger other applications to do something or to switch between activities in a single application.

To play audio back using an Intent, we can use the following code:

java.io.File audioFile = new File(android.os.Environment.getExternalStorageDirectory().getAbsolutePath() + "/blah/recording.3gp"); 
android.net.Uri audioFileUri = android.net.Uri.fromFile(audioFile);
  
android.content.Intent intent = new android.content.Intent(android.content.Intent.ACTION_VIEW); 
intent.setDataAndType(audioFileUri, "audio/mp3"); 
startActivity(intent);

This code assumes that an audio file called recording.3gp is available on the SD card of the device in a folder called “blah”. Unfortunately, we couldn’t use this code to play an audio file that we bundled with our application as the built in player wouldn’t be able to access the file (unless we moved it).

Also, as per a limitation in the current version of Processing for Android I am using entire package names for the classes I am using rather than normal “import” statements. Normally the imports would look like this:

import java.io.File;
import android.content.Intent;
import android.net.Uri;
import android.os.Environment;

and the code snippet would be:

File audioFile = new File(Environment.getExternalStorageDirectory().getAbsolutePath() + "/blah/recording.3gp"); 
Uri audioFileUri = Uri.fromFile(audioFile);
  
Intent intent = new Intent(Intent.ACTION_VIEW); 
intent.setDataAndType(audioFileUri, "audio/mp3"); 
startActivity(intent);

As with any code on Android, we can learn a lot about the individual classes we are using by referencing the “javadocs” on the Android Developer site.

File
Uri
Environment
Intent

Audio Capture using an Intent

android.net.Uri audioFileUri;
int RECORD_REQUEST = 0;

void setup() {
}

void draw() {
}

void mousePressed() {
  android.content.Intent intent = new android.content.Intent(android.provider.MediaStore.Audio.Media.RECORD_SOUND_ACTION); 
  startActivityForResult(intent, RECORD_REQUEST);
}

protected void onActivityResult (int requestCode, int resultCode, android.content.Intent data) { 
  if (resultCode == RESULT_OK && requestCode == RECORD_REQUEST) {
    audioFileUri = data.getData(); 
    println(audioFileUri);
  }
}

Audio Playback using MediaPlayer

Last week we looked at a class that leverages MediaPlayer to play audio back.

Here is an updated version of that class

class SoundPlayer extends Object  implements android.media.MediaPlayer.OnCompletionListener {

  android.media.MediaPlayer mediaPlayer;
  android.content.res.AssetFileDescriptor assetFileDescriptor;
  processing.core.PApplet theSketch;

  boolean ready = false;
 
  public SoundPlayer(PApplet _theSketch, String assetAudioFile) {
    theSketch = _theSketch;

    try {
      assetFileDescriptor = theSketch.getAssets().openFd(assetAudioFile);

      mediaPlayer = new android.media.MediaPlayer();
      mediaPlayer.setDataSource(assetFileDescriptor.getFileDescriptor(),assetFileDescriptor.getStartOffset(), assetFileDescriptor.getLength());
      mediaPlayer.setOnCompletionListener(this);
      mediaPlayer.prepare();
      ready = true;
    } catch (Exception e) {
      theSketch.println(e.toString());
    }
  }
 
  public SoundPlayer(PApplet _theSketch, java.io.File _audioFile) {
    this(_theSketch, android.net.Uri.fromFile(_audioFile));
  }
  
  public SoundPlayer(PApplet _theSketch, android.net.Uri _audioFileUri) {
    theSketch = _theSketch;
    theSketch.println("The Uri: " + _audioFileUri.toString());
    try {
      mediaPlayer = android.media.MediaPlayer.create(_theSketch, _audioFileUri);
      mediaPlayer.setOnCompletionListener(this);
      ready = true;
    } catch (Exception e) {
      theSketch.println(e.toString());
      e.printStackTrace();
      
    }
  }

  public void play() {
    if (ready) {
      try {
        theSketch.println("playSound");
        mediaPlayer.start();
        ready = false;
      } catch (Exception e) {
        theSketch.println(e.toString());
        e.printStackTrace();
      }
    }
  }

  public void onCompletion(android.media.MediaPlayer mp) {
    try {
      theSketch.println("Trying to prepare");
      mediaPlayer.stop();
      mediaPlayer.prepare();
      ready = true;
    }
    catch (Exception e) {
      theSketch.println(e.toString());
      e.printStackTrace();
    }
  }
}

Audio Capture using MediaRecorder

class SoundRecorder extends Object {

  android.media.MediaRecorder mediaRecorder;
  processing.core.PApplet theSketch;

  android.media.MediaRecorder recorder;
  boolean ready = false;
  java.io.File audioFile;

  public SoundRecorder(PApplet _theSketch, File _audioFile) {
    theSketch = _theSketch;

    recorder = new android.media.MediaRecorder();
    recorder.setAudioSource(android.media.MediaRecorder.AudioSource.MIC);
    recorder.setOutputFormat(android.media.MediaRecorder.OutputFormat.THREE_GPP);
    recorder.setAudioEncoder(android.media.MediaRecorder.AudioEncoder.AMR_NB);

    audioFile = _audioFile;
    recorder.setOutputFile(audioFile.getAbsolutePath());

    try {
      recorder.prepare();
    } catch (IllegalStateException e) {
      throw new RuntimeException("IllegalStateException on MediaRecorder.prepare", e);
    } catch (IOException e) {
      throw new RuntimeException("IOException on MediaRecorder.prepare", e);
    }

    ready = true;
  }
  
  public SoundRecorder(PApplet _theSketch, String audioFilePath) {
    theSketch = _theSketch;

    recorder = new android.media.MediaRecorder();
    recorder.setAudioSource(android.media.MediaRecorder.AudioSource.MIC);
    recorder.setOutputFormat(android.media.MediaRecorder.OutputFormat.THREE_GPP);
    recorder.setAudioEncoder(android.media.MediaRecorder.AudioEncoder.AMR_NB);

    java.io.File audioFile = new java.io.File(audioFilePath);    
    recorder.setOutputFile(audioFile.getAbsolutePath());

    try {
      recorder.prepare();
    } catch (IllegalStateException e) {
      throw new RuntimeException("IllegalStateException on MediaRecorder.prepare", e);
    } catch (IOException e) {
      throw new RuntimeException("IOException on MediaRecorder.prepare", e);
    }

    ready = true;
  }

  public void record() {
    if (ready) {
      recorder.start();    
    }
  }

  public void stop() {
     recorder.stop(); 
     recorder.release();
  }
}

Example:

int clickNumber = 0;

SoundPlayer sp;
SoundRecorder sr;

java.io.File recordingFile;

void setup() {
  java.io.File externalStorageDirectory = android.os.Environment.getExternalStorageDirectory();
  recordingFile = new java.io.File(externalStorageDirectory, "hi.3gp");
}

void draw() {
}

void mousePressed() {
  
  if (clickNumber == 0) {
    println("recording");      
    sr = new SoundRecorder(this, recordingFile);
    sr.record();
  } 
  else if (clickNumber == 1) {
    println("stop recording");

    sr.stop();
  } 
  else if (clickNumber == 2) {
    println("playback");

    sp = new SoundPlayer(this, recordingFile);
    sp.play();
  }

  clickNumber++;
}

Don’t forget the permissions: WRITE_EXTERNAL_STORAGE and RECORD_AUDIO

There are several ways to play audio on Android. One of the basic capabilities is to simply playback audio files that are saved with a Sketch. The below example uses a class called SoundPlayer (code below) to play an audio file (“beep.ogg“) that has been placed into the Sketch’s data folder.

SoundPlayer beepSound;
SoundPlayer otherSound;

void setup() {
  beepSound = new SoundPlayer(this,"beep.ogg");
  //otherSound = new SoundPlayer(this,"other.wav");
}

void draw() {
  
}

void mousePressed() {
  beepSound.play();
}

class SoundPlayer extends Object  implements android.media.MediaPlayer.OnCompletionListener {
  
  android.media.MediaPlayer mediaPlayer;
  android.content.res.AssetFileDescriptor assetFileDescriptor;
  processing.core.PApplet theSketch;
  
  boolean ready = false;
  
  public SoundPlayer(PApplet _theSketch, String audioFile) {
    theSketch = _theSketch;
    
    try {
      assetFileDescriptor = theSketch.getAssets().openFd(audioFile);
      
      mediaPlayer = new android.media.MediaPlayer();
      mediaPlayer.setDataSource(assetFileDescriptor.getFileDescriptor(),assetFileDescriptor.getStartOffset(), assetFileDescriptor.getLength());
      mediaPlayer.setOnCompletionListener(this);
      mediaPlayer.prepare();
      ready = true;
    } catch (Exception e) {
      println(e); 
    }
  }
  
  public void play() {
    if (ready) {
      try {
        mediaPlayer.start();
        ready = false;
        println("playSound");
      } catch (Exception e) {
        println(e); 
      }
    }
  }
  
  public void onCompletion(android.media.MediaPlayer mp) { 
    try {
      println("Trying to prepare");
      mediaPlayer.stop();
      mediaPlayer.prepare();
      ready = true;
    }
    catch (Exception e) {
      println(e); 
    }
  }  
}  

The SoundPlayer class uses the built-in MediaPlayer class which is the base class for video playback as well. We’ll dig deeper into the MediaPlayer in future classes.