diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java new file mode 100644 index 00000000..68f3e9f6 --- /dev/null +++ b/server/src/main/java/com/genymobile/scrcpy/AudioCodec.java @@ -0,0 +1,46 @@ +package com.genymobile.scrcpy; + +import android.media.MediaFormat; + +public enum AudioCodec implements Codec { + OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS); + + private final int id; // 4-byte ASCII representation of the name + private final String name; + private final String mimeType; + + AudioCodec(int id, String name, String mimeType) { + this.id = id; + this.name = name; + this.mimeType = mimeType; + } + + @Override + public Type getType() { + return Type.VIDEO; + } + + @Override + public int getId() { + return id; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getMimeType() { + return mimeType; + } + + public static AudioCodec findByName(String name) { + for (AudioCodec codec : values()) { + if (codec.name.equals(name)) { + return codec; + } + } + return null; + } +} diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 53fbf3ff..e3fd953d 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -4,17 +4,66 @@ import android.annotation.SuppressLint; import android.annotation.TargetApi; import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.AudioTimestamp; +import android.media.MediaCodec; +import android.media.MediaFormat; import android.media.MediaRecorder; import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.SystemClock; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; public final class AudioEncoder { + private static class InputTask { + final int index; + + InputTask(int index) { + this.index = index; + } + } + + private static class OutputTask { + final int index; + final MediaCodec.BufferInfo bufferInfo; + + OutputTask(int index, MediaCodec.BufferInfo bufferInfo) { + this.index = index; + this.bufferInfo = bufferInfo; + } + } + + private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS; private static final int SAMPLE_RATE = 48000; private static final int CHANNELS = 2; + private static final int BIT_RATE = 128000; + + private static int BUFFER_MS = 15; // milliseconds + private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000; + + private AudioRecord recorder; + private MediaCodec mediaCodec; + + private final AtomicBoolean cleanUpDone = new AtomicBoolean(false); + + // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4). + // So many pending tasks would lead to an unacceptable delay anyway. + private final BlockingQueue inputTasks = new ArrayBlockingQueue<>(64); + private final BlockingQueue outputTasks = new ArrayBlockingQueue<>(64); private Thread thread; + private HandlerThread mediaCodecThread; + + private Thread inputThread; + private Thread outputThread; + + private boolean ended; private static AudioFormat createAudioFormat() { AudioFormat.Builder builder = new AudioFormat.Builder(); @@ -38,25 +87,80 @@ public final class AudioEncoder { return builder.build(); } - public void start() { - AudioRecord recorder = createAudioRecord(); + private static MediaFormat createFormat() { + MediaFormat format = new MediaFormat(); + format.setString(MediaFormat.KEY_MIME, MIMETYPE); + format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS); + format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE); + return format; + } - thread = new Thread(() -> { - recorder.startRecording(); - try { - int BUFFER_MS = 15; // do not buffer more than BUFFER_MS milliseconds - byte[] buf = new byte[SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000]; - while (!Thread.currentThread().isInterrupted()) { - int r = recorder.read(buf, 0, buf.length); - if (r > 0) { - Ln.i("Audio captured: " + r + " bytes"); - } - if (r < 0) { - Ln.e("Audio capture error: " + r); - } + @TargetApi(Build.VERSION_CODES.N) + private void inputThread() throws IOException, InterruptedException { + final AudioTimestamp timestamp = new AudioTimestamp(); + long previousPts = 0; + long nextPts = 0; + + while (!Thread.currentThread().isInterrupted()) { + InputTask task = inputTasks.take(); + ByteBuffer buffer = mediaCodec.getInputBuffer(task.index); + int r = recorder.read(buffer, BUFFER_SIZE); + if (r < 0) { + throw new IOException("Could not read audio: " + r); + } + + long pts; + + int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + if (ret == AudioRecord.SUCCESS) { + pts = timestamp.nanoTime / 1000; + } else { + if (nextPts == 0) { + Ln.w("Could not get any audio timestamp"); } + // compute from previous timestamp and packet size + pts = nextPts; + } + + long durationMs = r * 1000 / CHANNELS / SAMPLE_RATE; + nextPts = pts + durationMs; + + if (previousPts != 0 && pts < previousPts) { + // Audio PTS may come from two sources: + // - recorder.getTimestamp() if the call works; + // - an estimation from the previous PTS and the packet size as a fallback. + // + // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it. + pts = previousPts + 1; + } + + mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0); + + previousPts = pts; + } + } + + private void outputThread() throws IOException, InterruptedException { + while (!Thread.currentThread().isInterrupted()) { + OutputTask task = outputTasks.take(); + ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index); + try { + Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes"); } finally { - recorder.stop(); + mediaCodec.releaseOutputBuffer(task.index, false); + } + } + } + + public void start() { + thread = new Thread(() -> { + try { + encode(); + } catch (IOException e) { + // this is expected on close + } finally { + Ln.d("Audio encoder stopped"); } }); thread.start(); @@ -64,7 +168,8 @@ public final class AudioEncoder { public void stop() { if (thread != null) { - thread.interrupt(); + // Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates + end(); } } @@ -73,4 +178,125 @@ public final class AudioEncoder { thread.join(); } } + + private synchronized void end() { + ended = true; + notify(); + } + + private synchronized void waitEnded() { + try { + while (!ended) { + wait(); + } + } catch (InterruptedException e) { + // ignore + } + } + + @TargetApi(Build.VERSION_CODES.M) + public void encode() throws IOException { + mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException + + try { + recorder = createAudioRecord(); + + mediaCodecThread = new HandlerThread("AudioEncoder"); + mediaCodecThread.start(); + + MediaFormat format = createFormat(); + mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper())); + mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + + recorder.startRecording(); + + inputThread = new Thread(() -> { + try { + inputThread(); + } catch (IOException | InterruptedException e) { + // this is expected on close + } finally { + end(); + } + }); + + outputThread = new Thread(() -> { + try { + outputThread(); + } catch (IOException | InterruptedException e) { + // this is expected on close + } finally { + end(); + } + }); + + mediaCodec.start(); + inputThread.start(); + outputThread.start(); + } catch (Throwable e) { + mediaCodec.release(); + if (recorder != null) { + recorder.release(); + } + throw e; + } + + try { + waitEnded(); + } finally { + cleanUp(); + } + } + + private void cleanUp() { + mediaCodecThread.getLooper().quit(); + inputThread.interrupt(); + outputThread.interrupt(); + + try { + mediaCodecThread.join(); + inputThread.join(); + outputThread.join(); + } catch (InterruptedException e) { + // Should never happen + throw new AssertionError(e); + } + + mediaCodec.stop(); + mediaCodec.release(); + recorder.stop(); + recorder.release(); + } + + private class EncoderCallback extends MediaCodec.Callback { + @TargetApi(Build.VERSION_CODES.N) + @Override + public void onInputBufferAvailable(MediaCodec codec, int index) { + try { + inputTasks.put(new InputTask(index)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) { + try { + outputTasks.put(new OutputTask(index, bufferInfo)); + } catch (InterruptedException e) { + end(); + } + } + + @Override + public void onError(MediaCodec codec, MediaCodec.CodecException e) { + Ln.e("MediaCodec error", e); + end(); + } + + @Override + public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { + // ignore + } + } }