add IME 'ScrcpyIME'

* add switch cmd 'ctrl+e' to enable and use ime, 'ctrl+shift+e' to disable ime
* change original handler behavior:
* change text_input to send all types char
* when enable IME, injectKeyEvent skip Letter or Digit or Space char, InjectText to handler it
* when disable IME, InjectText skip all text, injectKeyEvent to handler it
This commit is contained in:
pangliang 2019-11-15 18:06:33 +08:00
parent 771bd8404d
commit 1d3d8b2cc7
22 changed files with 420 additions and 22 deletions

3
.gitignore vendored
View file

@ -2,3 +2,6 @@ build/
/dist/
.idea/
.gradle/
x
*.iml
local.properties

View file

@ -80,6 +80,9 @@ control_msg_serialize(const struct control_msg *msg, unsigned char *buf) {
case CONTROL_MSG_TYPE_GET_CLIPBOARD:
// no additional data
return 1;
case CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE:
buf[1] = msg->set_inject_text_mode.mode;
return 2;
default:
LOGW("Unknown message type: %u", (unsigned) msg->type);
return 0;

View file

@ -28,6 +28,7 @@ enum control_msg_type {
CONTROL_MSG_TYPE_GET_CLIPBOARD,
CONTROL_MSG_TYPE_SET_CLIPBOARD,
CONTROL_MSG_TYPE_SET_SCREEN_POWER_MODE,
CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE,
};
enum screen_power_mode {
@ -36,6 +37,11 @@ enum screen_power_mode {
SCREEN_POWER_MODE_NORMAL = 2,
};
enum inject_text_mode {
USE_INPUT_MANAGER = 0,
USE_SCRCPY_IME = 1,
};
struct control_msg {
enum control_msg_type type;
union {
@ -65,6 +71,9 @@ struct control_msg {
struct {
enum screen_power_mode mode;
} set_screen_power_mode;
struct {
enum inject_text_mode mode;
} set_inject_text_mode;
};
};

View file

@ -173,6 +173,18 @@ set_screen_power_mode(struct controller *controller,
}
}
static void
set_inject_text_mode(struct controller *controller,
enum inject_text_mode mode) {
struct control_msg msg;
msg.type = CONTROL_MSG_TYPE_SET_INJECT_TEXT_MODE;
msg.set_inject_text_mode.mode = mode;
if (!controller_push_msg(controller, &msg)) {
LOGW("Could not request 'set screen power mode'");
}
}
static void
switch_fps_counter_state(struct fps_counter *fps_counter) {
// the started state can only be written from the current thread, so there
@ -388,6 +400,15 @@ input_manager_process_key(struct input_manager *im,
}
}
return;
case SDLK_e:
if (control && cmd && !repeat && down) {
if (shift) {
set_inject_text_mode(controller, USE_INPUT_MANAGER);
} else {
set_inject_text_mode(controller, USE_SCRCPY_IME);
}
}
return;
}
return;

View file

@ -146,12 +146,16 @@ handle_event(SDL_Event *event, bool control) {
case SDL_WINDOWEVENT:
screen_handle_window_event(&screen, &event->window);
break;
case SDL_TEXTEDITING:
break;
case SDL_TEXTINPUT:
if (!control) {
break;
}
input_manager_process_text_input(&input_manager, &event->text);
break;
case SDL_KEYMAPCHANGED:
break;
case SDL_KEYDOWN:
case SDL_KEYUP:
// some key events do not interact with the device, so process the
@ -195,6 +199,8 @@ handle_event(SDL_Event *event, bool control) {
file_handler_request(&file_handler, action, event->drop.file);
break;
}
default:
LOGD("Unknow event type:%d", event->type);
}
return EVENT_RESULT_CONTINUE;
}

28
ime/build.gradle Normal file
View file

@ -0,0 +1,28 @@
apply plugin: 'com.android.application'
android {
compileSdkVersion 29
defaultConfig {
applicationId "com.genymobile.scrcpy.ime"
minSdkVersion 21
targetSdkVersion 29
versionCode 1
versionName "1.0"
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
}
buildTypes {
release {
minifyEnabled false
proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
}
}
}
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
testImplementation 'junit:junit:4.12'
}

21
ime/proguard-rules.pro vendored Normal file
View file

@ -0,0 +1,21 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html
# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}
# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable
# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile

View file

@ -0,0 +1,10 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.genymobile.scrcpy.ime">
<application android:label="@string/app_name">
<service android:label="@string/app_name" android:name="com.genymobile.scrcpy.ime.ScrcpyIME" android:permission="android.permission.BIND_INPUT_METHOD">
<intent-filter>
<action android:name="android.view.InputMethod"/>
</intent-filter>
<meta-data android:name="android.view.im" android:resource="@xml/method" />
</service>
</application>
</manifest>

View file

@ -0,0 +1,81 @@
package com.genymobile.scrcpy.ime;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.content.IntentFilter;
import android.inputmethodservice.InputMethodService;
import android.util.Log;
import android.view.KeyEvent;
import android.view.inputmethod.InputConnection;
public class ScrcpyIME extends InputMethodService {
private static String TAG = "ScrcpyIME";
private BroadcastReceiver receiver;
private static final String COMMIT_TEXT_ACTION = "com.genymobile.scrcpy.ime.COMMIT_TEXT_ACTION";
private static final String STATE_CHANGE_ACTION = "com.genymobile.scrcpy.ime.STATE_CHANGE_ACTION";
@Override
public void onCreate() {
super.onCreate();
this.receiver = new BroadcastReceiver() {
@Override
public void onReceive(Context context, Intent intent) {
InputConnection inputConnection = getCurrentInputConnection();
if(inputConnection == null){
return;
}
String text = null;
KeyEvent keyEvent = null;
if((text = intent.getStringExtra("text")) != null && text.length() > 0) {
inputConnection.commitText(text, 0);
}else if((keyEvent = intent.getParcelableExtra("keyEvent")) != null) {
inputConnection.sendKeyEvent(keyEvent);
}
}
};
IntentFilter localIntentFilter = new IntentFilter(COMMIT_TEXT_ACTION);
registerReceiver(this.receiver, localIntentFilter);
}
@Override
public void onDestroy() {
unregisterReceiver(this.receiver);
Log.i(TAG, "disabling self due to destroy");
super.onDestroy();
}
@Override
public void onBindInput() {
super.onBindInput();
Log.i(TAG, "BindInput");
sendStateBroadcast("BindInput");
}
@Override
public void onUnbindInput() {
super.onUnbindInput();
Log.i(TAG, "UnbindInput");
sendStateBroadcast("UnbindInput");
}
@Override
public void onWindowShown() {
super.onWindowShown();
Log.i(TAG, "WindowShown");
sendStateBroadcast("WindowShown");
}
@Override
public void onWindowHidden() {
super.onWindowHidden();
Log.i(TAG, "WindowHidden");
sendStateBroadcast("WindowHidden");
}
private void sendStateBroadcast(String state) {
Intent intent = new Intent(STATE_CHANGE_ACTION);
intent.putExtra("state", state);
sendBroadcast(intent);
}
}

View file

@ -0,0 +1,3 @@
<resources>
<string name="app_name">Scrcpy</string>
</resources>

View file

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<input-method xmlns:android="http://schemas.android.com/apk/res/android">
<subtype android:label="@string/app_name" android:imeSubtypeLocale="en_US" android:imeSubtypeMode="keyboard" />
</input-method>

View file

@ -3,7 +3,7 @@
prebuilt_server = get_option('prebuilt_server')
if prebuilt_server == ''
custom_target('scrcpy-server',
build_always: true, # gradle is responsible for tracking source changes
build_by_default: true, # gradle is responsible for tracking source changes
output: 'scrcpy-server',
command: [find_program('./scripts/build-wrapper.sh'), meson.current_source_dir(), '@OUTPUT@', get_option('buildtype')],
console: true,

View file

@ -0,0 +1,29 @@
/*
* Copyright (C) 2006 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package android.content;
import android.content.Intent;
import android.os.Bundle;
/**
* System private API for dispatching intent broadcasts. This is given to the
* activity manager as part of registering for an intent broadcasts, and is
* called when it receives intents.
*
* {@hide}
*/
oneway interface IIntentReceiver {
void performReceive(in Intent intent, int resultCode, String data,
in Bundle extras, boolean ordered, boolean sticky, int sendingUser);
}

View file

@ -0,0 +1,23 @@
/*
* Copyright (C) 2008 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.internal.view;
/**
* Interface a client of the IInputMethodManager implements, to identify
* itself and receive information about changes to the global manager state.
*/
interface IInputMethodClient {
}

View file

@ -15,6 +15,7 @@ public final class ControlMessage {
public static final int TYPE_GET_CLIPBOARD = 7;
public static final int TYPE_SET_CLIPBOARD = 8;
public static final int TYPE_SET_SCREEN_POWER_MODE = 9;
public static final int TYPE_SET_INJECT_TEXT_MODE = 10;
private int type;
private String text;
@ -85,6 +86,13 @@ public final class ControlMessage {
return msg;
}
public static ControlMessage createSetInjectTextMode(int mode) {
ControlMessage msg = new ControlMessage();
msg.type = TYPE_SET_INJECT_TEXT_MODE;
msg.action = mode;
return msg;
}
public static ControlMessage createEmpty(int type) {
ControlMessage msg = new ControlMessage();
msg.type = type;

View file

@ -13,6 +13,7 @@ public class ControlMessageReader {
private static final int INJECT_TOUCH_EVENT_PAYLOAD_LENGTH = 21;
private static final int INJECT_SCROLL_EVENT_PAYLOAD_LENGTH = 20;
private static final int SET_SCREEN_POWER_MODE_PAYLOAD_LENGTH = 1;
private static final int SET_INJECT_TEXT_MODE_PAYLOAD_LENGTH = 1;
public static final int TEXT_MAX_LENGTH = 300;
public static final int CLIPBOARD_TEXT_MAX_LENGTH = 4093;
@ -78,6 +79,9 @@ public class ControlMessageReader {
case ControlMessage.TYPE_GET_CLIPBOARD:
msg = ControlMessage.createEmpty(type);
break;
case ControlMessage.TYPE_SET_INJECT_TEXT_MODE:
msg = parseSetInjectTextMode();
break;
default:
Ln.w("Unknown event type: " + type);
msg = null;
@ -163,6 +167,14 @@ public class ControlMessageReader {
return ControlMessage.createSetScreenPowerMode(mode);
}
private ControlMessage parseSetInjectTextMode() {
if (buffer.remaining() < SET_INJECT_TEXT_MODE_PAYLOAD_LENGTH) {
return null;
}
int mode = buffer.get();
return ControlMessage.createSetInjectTextMode(mode);
}
private static Position readPosition(ByteBuffer buffer) {
int x = buffer.getInt();
int y = buffer.getInt();

View file

@ -1,15 +1,15 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.InputManager;
import java.io.IOException;
import android.content.Intent;
import android.os.SystemClock;
import android.view.InputDevice;
import android.view.InputEvent;
import android.view.KeyCharacterMap;
import android.view.KeyEvent;
import android.view.MotionEvent;
import java.io.IOException;
import com.genymobile.scrcpy.wrappers.InputManager;
public class Controller {
@ -26,6 +26,9 @@ public class Controller {
private final MotionEvent.PointerCoords[] pointerCoords =
new MotionEvent.PointerCoords[PointersState.MAX_POINTERS];
private static final String COMMIT_TEXT_ACTION = "com.genymobile.scrcpy.ime.COMMIT_TEXT_ACTION";
private boolean useIME = true;
public Controller(Device device, DesktopConnection connection) {
this.device = device;
this.connection = connection;
@ -76,7 +79,7 @@ public class Controller {
ControlMessage msg = connection.receiveControlMessage();
switch (msg.getType()) {
case ControlMessage.TYPE_INJECT_KEYCODE:
injectKeycode(msg.getAction(), msg.getKeycode(), msg.getMetaState());
injectKeyEvent(msg.getAction(), msg.getKeycode(), 0, msg.getMetaState());
break;
case ControlMessage.TYPE_INJECT_TEXT:
injectText(msg.getText());
@ -106,15 +109,15 @@ public class Controller {
case ControlMessage.TYPE_SET_SCREEN_POWER_MODE:
device.setScreenPowerMode(msg.getAction());
break;
case ControlMessage.TYPE_SET_INJECT_TEXT_MODE:
useIME = msg.getAction() == 1;
device.setInjectTextMode(useIME);
break;
default:
// do nothing
}
}
private boolean injectKeycode(int action, int keycode, int metaState) {
return injectKeyEvent(action, keycode, 0, metaState);
}
private boolean injectChar(char c) {
String decomposed = KeyComposition.decompose(c);
char[] chars = decomposed != null ? decomposed.toCharArray() : new char[]{c};
@ -130,16 +133,14 @@ public class Controller {
return true;
}
private int injectText(String text) {
int successCount = 0;
for (char c : text.toCharArray()) {
if (!injectChar(c)) {
Ln.w("Could not inject char u+" + String.format("%04x", (int) c));
continue;
}
successCount++;
private void injectText(String text) {
if(!useIME) {
return;
}
return successCount;
Intent intent = new Intent();
intent.setAction(COMMIT_TEXT_ACTION);
intent.putExtra("text", text);
device.sendBroadcast(intent);
}
private boolean injectTouch(int action, long pointerId, Position position, float pressure, int buttons) {
@ -204,6 +205,11 @@ public class Controller {
}
private boolean injectKeyEvent(int action, int keyCode, int repeat, int metaState) {
char keyChar = (char)charMap.get(keyCode,metaState);
if(useIME
&& (Character.isLetterOrDigit(keyChar) || ' ' == keyChar)) {
return true;
}
long now = SystemClock.uptimeMillis();
KeyEvent event = new KeyEvent(now, now, action, keyCode, repeat, metaState, KeyCharacterMap.VIRTUAL_KEYBOARD,
0, 0, InputDevice.SOURCE_KEYBOARD);

View file

@ -1,14 +1,14 @@
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.ServiceManager;
import com.genymobile.scrcpy.wrappers.SurfaceControl;
import android.content.Intent;
import android.graphics.Rect;
import android.os.Build;
import android.os.IBinder;
import android.os.RemoteException;
import android.view.IRotationWatcher;
import android.view.InputEvent;
import com.genymobile.scrcpy.wrappers.ServiceManager;
import com.genymobile.scrcpy.wrappers.SurfaceControl;
public final class Device {
@ -157,6 +157,19 @@ public final class Device {
Ln.i("Device clipboard set");
}
public void showInputMethodPicker() {
serviceManager.getInputMethodManager().showInputMethodPicker();
Ln.i("Input Method Picker show");
}
public void sendBroadcast(Intent intent) {
serviceManager.getActivityManager().sendBroadcast(intent);
}
public void setInjectTextMode(boolean useIME) {
serviceManager.getInputMethodManager().showInputMethodPicker();
}
/**
* @param mode one of the {@code SCREEN_POWER_MODE_*} constants
*/

View file

@ -0,0 +1,66 @@
package com.genymobile.scrcpy.wrappers;
import java.lang.reflect.Method;
import android.annotation.SuppressLint;
import android.content.IIntentReceiver;
import android.content.Intent;
import android.content.IntentFilter;
import android.os.Bundle;
import android.os.IInterface;
import android.os.RemoteException;
import com.genymobile.scrcpy.Ln;
@SuppressLint("PrivateApi")
public class ActivityManager {
private final IInterface manager;
private Method broadcastIntentMethod;
private Method registerReceiverMethod;
public ActivityManager(IInterface manager) {
this.manager = manager;
try {
for (Method method : manager.getClass().getDeclaredMethods()) {
if (method.getName().equals("broadcastIntent")) {
int parameterLength = method.getParameterTypes().length;
if (parameterLength != 13 && parameterLength != 12 && parameterLength != 11) {
Ln.i("broadcastIntent method parameter length wrong.");
continue;
}
broadcastIntentMethod = method;
}else if(method.getName().equals("registerReceiver")) {
registerReceiverMethod = method;
}
}
} catch (Exception e) {
throw new AssertionError(e);
}
}
public void sendBroadcast(Intent paramIntent) {
try {
if (broadcastIntentMethod.getParameterTypes().length == 11) {
broadcastIntentMethod.invoke(
manager, null, paramIntent, null, null, 0, null, null, null, Boolean.TRUE, Boolean.FALSE, -2);
} else if (broadcastIntentMethod.getParameterTypes().length == 12) {
broadcastIntentMethod.invoke(
manager, null, paramIntent, null, null, 0, null, null, null, -1, Boolean.TRUE, Boolean.FALSE, -2);
} else if (broadcastIntentMethod.getParameterTypes().length == 13) {
broadcastIntentMethod.invoke(
manager, null, paramIntent, null, null, 0, null, null, null, -1, null, Boolean.TRUE, Boolean.FALSE, -2);
}
}catch (Exception e){
throw new AssertionError(e);
}
}
//Currently not working
public Intent registerReceiver(IIntentReceiver receiver, IntentFilter intentFilter) {
try {
return (Intent)registerReceiverMethod.invoke(manager, null, null, receiver, intentFilter, null, -2, 0);
}catch (Exception e){
throw new AssertionError(e);
}
}
}

View file

@ -0,0 +1,35 @@
package com.genymobile.scrcpy.wrappers;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import android.os.IInterface;
import com.android.internal.view.IInputMethodClient;
import com.genymobile.scrcpy.Ln;
public final class InputMethodManager {
private final IInterface manager;
private final Method showInputMethodPickerMethod;
private final IInputMethodClient.Stub stub = new IInputMethodClient.Stub() {};
public InputMethodManager(IInterface manager) {
this.manager = manager;
try {
for(Field field : manager.getClass().getDeclaredFields()) {
Ln.i("field:" + field.getName());
}
showInputMethodPickerMethod = manager.getClass().getMethod("showInputMethodPickerFromClient", IInputMethodClient.class, int.class);
} catch (NoSuchMethodException e) {
throw new AssertionError(e);
}
}
public void showInputMethodPicker() {
try {
showInputMethodPickerMethod.invoke(manager, stub, 0);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new AssertionError(e);
}
}
}

View file

@ -3,6 +3,7 @@ package com.genymobile.scrcpy.wrappers;
import android.annotation.SuppressLint;
import android.os.IBinder;
import android.os.IInterface;
import com.genymobile.scrcpy.Ln;
import java.lang.reflect.Method;
@ -16,6 +17,8 @@ public final class ServiceManager {
private PowerManager powerManager;
private StatusBarManager statusBarManager;
private ClipboardManager clipboardManager;
private InputMethodManager inputMethodManager;
private ActivityManager activityManager;
public ServiceManager() {
try {
@ -76,4 +79,18 @@ public final class ServiceManager {
}
return clipboardManager;
}
public InputMethodManager getInputMethodManager() {
if(inputMethodManager == null) {
inputMethodManager = new InputMethodManager(getService("input_method", "com.android.internal.view.IInputMethodManager"));
}
return inputMethodManager;
}
public ActivityManager getActivityManager() {
if(activityManager == null) {
activityManager = new ActivityManager(getService("activity", "android.app.IActivityManager"));
}
return activityManager;
}
}

View file

@ -1 +1 @@
include ':server'
include ':server', ':ime'