mirror of
https://github.com/Genymobile/scrcpy.git
synced 2025-04-22 04:25:01 +00:00
Encode recorded audio in OPUS on the device
For now, the encoded packets are just logged in the console.
This commit is contained in:
Romain Vimont
parent
bc0f51023e
commit
c19fefbcbd
1 changed files with 244 additions and 16 deletions
|
|
@ -4,17 +4,62 @@ import android.annotation.SuppressLint;
|
|||
import android.annotation.TargetApi;
|
||||
import android.media.AudioFormat;
|
||||
import android.media.AudioRecord;
|
||||
import android.media.AudioTimestamp;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaFormat;
|
||||
import android.media.MediaRecorder;
|
||||
import android.os.Build;
|
||||
import android.os.Handler;
|
||||
import android.os.HandlerThread;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
public final class AudioEncoder {
|
||||
|
||||
private static class InputTask {
|
||||
final int index;
|
||||
|
||||
InputTask(int index) {
|
||||
this.index = index;
|
||||
}
|
||||
}
|
||||
|
||||
private static class OutputTask {
|
||||
final int index;
|
||||
final MediaCodec.BufferInfo bufferInfo;
|
||||
|
||||
OutputTask(int index, MediaCodec.BufferInfo bufferInfo) {
|
||||
this.index = index;
|
||||
this.bufferInfo = bufferInfo;
|
||||
}
|
||||
}
|
||||
|
||||
private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
|
||||
private static final int SAMPLE_RATE = 48000;
|
||||
private static final int CHANNELS = 2;
|
||||
private static final int BIT_RATE = 196000;
|
||||
|
||||
private static final int BUFFER_MS = 15; // milliseconds
|
||||
private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;
|
||||
|
||||
private AudioRecord recorder;
|
||||
private MediaCodec mediaCodec;
|
||||
|
||||
// Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4).
|
||||
// So many pending tasks would lead to an unacceptable delay anyway.
|
||||
private final BlockingQueue<InputTask> inputTasks = new ArrayBlockingQueue<>(64);
|
||||
private final BlockingQueue<OutputTask> outputTasks = new ArrayBlockingQueue<>(64);
|
||||
|
||||
private Thread thread;
|
||||
private HandlerThread mediaCodecThread;
|
||||
|
||||
private Thread inputThread;
|
||||
private Thread outputThread;
|
||||
|
||||
private boolean ended;
|
||||
|
||||
private static AudioFormat createAudioFormat() {
|
||||
AudioFormat.Builder builder = new AudioFormat.Builder();
|
||||
|
|
@ -38,24 +83,80 @@ public final class AudioEncoder {
|
|||
return builder.build();
|
||||
}
|
||||
|
||||
public void start() {
|
||||
AudioRecord recorder = createAudioRecord();
|
||||
private static MediaFormat createFormat() {
|
||||
MediaFormat format = new MediaFormat();
|
||||
format.setString(MediaFormat.KEY_MIME, MIMETYPE);
|
||||
format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
|
||||
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
|
||||
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
|
||||
return format;
|
||||
}
|
||||
|
||||
thread = new Thread(() -> {
|
||||
recorder.startRecording();
|
||||
try {
|
||||
int BUFFER_MS = 15; // do not buffer more than BUFFER_MS milliseconds
|
||||
byte[] buf = new byte[SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000];
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
int r = recorder.read(buf, 0, buf.length);
|
||||
if (r > 0) {
|
||||
Ln.i("Audio captured: " + r + " bytes");
|
||||
} else {
|
||||
Ln.e("Audio capture error: " + r);
|
||||
}
|
||||
@TargetApi(Build.VERSION_CODES.N)
|
||||
private void inputThread() throws IOException, InterruptedException {
|
||||
final AudioTimestamp timestamp = new AudioTimestamp();
|
||||
long previousPts = 0;
|
||||
long nextPts = 0;
|
||||
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
InputTask task = inputTasks.take();
|
||||
ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
|
||||
int r = recorder.read(buffer, BUFFER_SIZE);
|
||||
if (r < 0) {
|
||||
throw new IOException("Could not read audio: " + r);
|
||||
}
|
||||
|
||||
long pts;
|
||||
|
||||
int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
|
||||
if (ret == AudioRecord.SUCCESS) {
|
||||
pts = timestamp.nanoTime / 1000;
|
||||
} else {
|
||||
if (nextPts == 0) {
|
||||
Ln.w("Could not get any audio timestamp");
|
||||
}
|
||||
// compute from previous timestamp and packet size
|
||||
pts = nextPts;
|
||||
}
|
||||
|
||||
long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE;
|
||||
nextPts = pts + durationMs;
|
||||
|
||||
if (previousPts != 0 && pts < previousPts) {
|
||||
// Audio PTS may come from two sources:
|
||||
// - recorder.getTimestamp() if the call works;
|
||||
// - an estimation from the previous PTS and the packet size as a fallback.
|
||||
//
|
||||
// Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
|
||||
pts = previousPts + 1;
|
||||
}
|
||||
|
||||
mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
|
||||
|
||||
previousPts = pts;
|
||||
}
|
||||
}
|
||||
|
||||
private void outputThread() throws IOException, InterruptedException {
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
OutputTask task = outputTasks.take();
|
||||
ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index);
|
||||
try {
|
||||
Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes");
|
||||
} finally {
|
||||
recorder.stop();
|
||||
mediaCodec.releaseOutputBuffer(task.index, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void start() {
|
||||
thread = new Thread(() -> {
|
||||
try {
|
||||
encode();
|
||||
} catch (IOException e) {
|
||||
Ln.e("Audio encoding error", e);
|
||||
} finally {
|
||||
Ln.d("Audio encoder stopped");
|
||||
}
|
||||
});
|
||||
thread.start();
|
||||
|
|
@ -63,7 +164,8 @@ public final class AudioEncoder {
|
|||
|
||||
public void stop() {
|
||||
if (thread != null) {
|
||||
thread.interrupt();
|
||||
// Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates
|
||||
end();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -72,4 +174,130 @@ public final class AudioEncoder {
|
|||
thread.join();
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void end() {
|
||||
ended = true;
|
||||
notify();
|
||||
}
|
||||
|
||||
private synchronized void waitEnded() {
|
||||
try {
|
||||
while (!ended) {
|
||||
wait();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
@TargetApi(Build.VERSION_CODES.M)
|
||||
public void encode() throws IOException {
|
||||
mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
|
||||
|
||||
try {
|
||||
recorder = createAudioRecord();
|
||||
|
||||
mediaCodecThread = new HandlerThread("AudioEncoder");
|
||||
mediaCodecThread.start();
|
||||
|
||||
MediaFormat format = createFormat();
|
||||
mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
|
||||
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
|
||||
|
||||
recorder.startRecording();
|
||||
|
||||
inputThread = new Thread(() -> {
|
||||
try {
|
||||
inputThread();
|
||||
} catch (IOException | InterruptedException e) {
|
||||
Ln.e("Audio capture error", e);
|
||||
} finally {
|
||||
end();
|
||||
}
|
||||
});
|
||||
|
||||
outputThread = new Thread(() -> {
|
||||
try {
|
||||
outputThread();
|
||||
} catch (InterruptedException e) {
|
||||
// this is expected on close
|
||||
} catch (IOException e) {
|
||||
// Broken pipe is expected on close, because the socket is closed by the client
|
||||
if (!IO.isBrokenPipe(e)) {
|
||||
Ln.e("Audio encoding error", e);
|
||||
}
|
||||
} finally {
|
||||
end();
|
||||
}
|
||||
});
|
||||
|
||||
mediaCodec.start();
|
||||
inputThread.start();
|
||||
outputThread.start();
|
||||
} catch (Throwable e) {
|
||||
mediaCodec.release();
|
||||
if (recorder != null) {
|
||||
recorder.release();
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
try {
|
||||
waitEnded();
|
||||
} finally {
|
||||
cleanUp();
|
||||
}
|
||||
}
|
||||
|
||||
private void cleanUp() {
|
||||
mediaCodecThread.getLooper().quit();
|
||||
inputThread.interrupt();
|
||||
outputThread.interrupt();
|
||||
|
||||
try {
|
||||
mediaCodecThread.join();
|
||||
inputThread.join();
|
||||
outputThread.join();
|
||||
} catch (InterruptedException e) {
|
||||
// Should never happen
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
|
||||
mediaCodec.stop();
|
||||
mediaCodec.release();
|
||||
recorder.stop();
|
||||
recorder.release();
|
||||
}
|
||||
|
||||
private class EncoderCallback extends MediaCodec.Callback {
|
||||
@TargetApi(Build.VERSION_CODES.N)
|
||||
@Override
|
||||
public void onInputBufferAvailable(MediaCodec codec, int index) {
|
||||
try {
|
||||
inputTasks.put(new InputTask(index));
|
||||
} catch (InterruptedException e) {
|
||||
end();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
|
||||
try {
|
||||
outputTasks.put(new OutputTask(index, bufferInfo));
|
||||
} catch (InterruptedException e) {
|
||||
end();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
|
||||
Ln.e("MediaCodec error", e);
|
||||
end();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue