Workaround to find UiObject2 and invoke actions

espresso-server/app/src/androidTest/java/io/appium/espressoserver/lib/handlers/Uiautomator.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.appium.espressoserver.lib.handlers;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
+import androidx.test.uiautomator.By;
+import androidx.test.uiautomator.BySelector;
+import androidx.test.uiautomator.UiObject2;
+import io.appium.espressoserver.lib.handlers.exceptions.AppiumException;
+import io.appium.espressoserver.lib.helpers.ReflectionUtils;
+import io.appium.espressoserver.lib.model.UiautomatorParams;
+
+import static io.appium.espressoserver.lib.helpers.AndroidLogger.logger;
+import static io.appium.espressoserver.lib.helpers.InteractionHelper.getUiDevice;
+
+public class Uiautomator implements RequestHandler<UiautomatorParams, List<Object>> {
+
+    @Override
+    public List<Object> handle(UiautomatorParams params) throws AppiumException {
+        logger.info("Invoking Uiautomator2 Methods");
+
+        if (null == params.getStrategy()) {
+            throw new AppiumException(String.format("strategy should be one of %s",
+                    UiautomatorParams.Strategy.getValidStrategyNames()));
+        }
+
+        if (null == params.getAction()) {
+            throw new AppiumException(String.format("strategy should be one of %s",
+                    UiautomatorParams.Action.getValidActionNames()));
+        }
+
+        String value = params.getValue();
+        Integer index = params.getIndex();
+
+        try {
+            Method byMethod = ReflectionUtils.method(By.class, params.getStrategy().getName(), String.class);
+            BySelector bySelector = (BySelector) ReflectionUtils.invoke(byMethod, By.class, value);
+            Method uiObjectMethod = ReflectionUtils.method(UiObject2.class, params.getAction().getName());
+            List<UiObject2> uiObjects = getUiDevice().findObjects(bySelector);
+            logger.info(String.format("Found %d UiObjects", uiObjects.size()));
+
+            List<Object> result = new ArrayList<>();
+
+            if (index == null) {
+                for (UiObject2 uiObject2 : uiObjects) {
+                    result.add(uiObjectMethod.invoke(uiObject2));
+                }
+                return result;
+            }
+
+            if (index >= uiObjects.size()) {
+                throw new AppiumException(
+                        String.format("Index %d is out of bounds for %d elements", index, uiObjects.size()));
+            }
+
+            result.add(uiObjectMethod.invoke(uiObjects.get(index)));
+            return result;
+        } catch (IllegalAccessException | InvocationTargetException e) {
+            throw new AppiumException(e);
+        }
+    }
+}

espresso-server/app/src/androidTest/java/io/appium/espressoserver/lib/http/Router.java

@@ -83,6 +83,7 @@
 import io.appium.espressoserver.lib.handlers.Text;
 import io.appium.espressoserver.lib.handlers.TouchAction;
 import io.appium.espressoserver.lib.handlers.TouchActionsParams;
+import io.appium.espressoserver.lib.handlers.Uiautomator;
 import io.appium.espressoserver.lib.handlers.exceptions.InvalidArgumentException;
 import io.appium.espressoserver.lib.handlers.exceptions.InvalidElementStateException;
 import io.appium.espressoserver.lib.handlers.exceptions.InvalidStrategyException;
@@ -120,6 +121,7 @@
 import io.appium.espressoserver.lib.model.StartActivityParams;
 import io.appium.espressoserver.lib.model.TextParams;
 import io.appium.espressoserver.lib.model.ToastLookupParams;
+import io.appium.espressoserver.lib.model.UiautomatorParams;
 import io.appium.espressoserver.lib.model.ViewFlashParams;
 
 import static io.appium.espressoserver.lib.handlers.PointerEventHandler.TouchType.CLICK;
@@ -223,6 +225,7 @@ class Router {
         routeMap.addRoute(new RouteDefinition(Method.POST, "/session/:sessionId/appium/execute_mobile/:elementId/navigate_to", new NavigateTo(), NavigateToParams.class));
         routeMap.addRoute(new RouteDefinition(Method.POST, "/session/:sessionId/appium/execute_mobile/backdoor", new MobileBackdoor(), MobileBackdoorParams.class));
         routeMap.addRoute(new RouteDefinition(Method.POST, "/session/:sessionId/appium/execute_mobile/:elementId/flash", new MobileViewFlash(), ViewFlashParams.class));
+        routeMap.addRoute(new RouteDefinition(Method.POST, "/session/:sessionId/appium/execute_mobile/uiautomator", new Uiautomator(), UiautomatorParams.class));
 
         // Not implemented
         routeMap.addRoute(new RouteDefinition(Method.POST, "/session/:sessionId/touch/flick", new NotYetImplemented(), AppiumParams.class));

espresso-server/app/src/androidTest/java/io/appium/espressoserver/lib/model/UiautomatorParams.java

@@ -0,0 +1,184 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * See the NOTICE file distributed with this work for additional
+ * information regarding copyright ownership.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.appium.espressoserver.lib.model;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+public class UiautomatorParams extends AppiumParams {
+    private Strategy strategy;
+    private String value;
+    private Integer index;
+    private Action action;
+
+    @androidx.annotation.Nullable
+    public Action getAction() {
+        return action;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    @androidx.annotation.Nullable
+    public Strategy getStrategy() {
+        return strategy;
+    }
+
+    @Nullable
+    public Integer getIndex() {
+        return index;
+    }
+
+    public enum Strategy {
+        @SerializedName("clazz")
+        CLASS_NAME("clazz"),
+
+        @SerializedName("res")
+        ID("res"),
+
+        @SerializedName("text")
+        TEXT("text"),
+
+        @SerializedName("textContains")
+        TEXT_CONTAIN("textContains"),
+
+        @SerializedName("textEndsWith")
+        TEXT_ENDS_WITH("textEndsWith"),
+
+        @SerializedName("textStartsWith")
+        TEXT_STARTS_WITH("textStartsWith"),
+
+        @SerializedName("desc")
+        DESC("desc"),
+
+        @SerializedName("descContains")
+        DESC_CONTAINS("descContains"),
+
+        @SerializedName("descEndsWith")
+        DESC_ENDS_WITH("descEndsWith"),
+
+        @SerializedName("descStartsWith")
+        DESC_STARTS_WITH("descStartsWith"),
+
+        @SerializedName("pkg")
+        APPLICATION_PACKAGE("pkg"),;
+
+        private final String name;
+
+        Strategy(final String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public static List<String> getValidStrategyNames() {
+            List<String> validStrategies = new ArrayList<>();
+            for (Strategy strategy : Strategy.values()) {
+                validStrategies.add(strategy.getName());
+            }
+            return validStrategies;
+        }
+    }
+
+
+    public enum Action {
+        @SerializedName("click")
+        CLICK("click"),
+
+        @SerializedName("longClick")
+        LONG_CLICK("longClick"),
+
+        @SerializedName("getText")
+        GET_TEXT("getText"),
+
+        @SerializedName("getContentDescription")
+        GET_CONTENT_DESCRIPTION("getContentDescription"),
+
+        @SerializedName("getClassName")
+        GET_CLASS_NAME("getClassName"),
+
+        @SerializedName("getResourceName")
+        GET_RESOURCE_NAME("getResourceName"),
+
+        @SerializedName("getVisibleBounds")
+        GET_VISIBLE_BOUNDS("getVisibleBounds"),
+
+        @SerializedName("getVisibleCenter")
+        GET_VISIBLE_CENTER("getVisibleCenter"),
+
+        @SerializedName("getApplicationPackage")
+        GET_APPLICATION_PACKAGE("getApplicationPackage"),
+
+        @SerializedName("getChildCount")
+        GET_CHILD_COUNT("getChildCount"),
+
+        @SerializedName("clear")
+        CLEAR("clear"),
+
+        @SerializedName("isCheckable")
+        IS_CHECKABLE("isCheckable"),
+
+        @SerializedName("isChecked")
+        IS_CHECKED("isChecked"),
+
+        @SerializedName("isClickable")
+        IS_CLICKABLE("isClickable"),
+
+        @SerializedName("isEnabled")
+        IS_ENABLED("isEnabled"),
+
+        @SerializedName("isFocusable")
+        IS_FOCUSABLE("isFocusable"),
+
+        @SerializedName("isFocused")
+        IS_FOCUSED("isFocused"),
+
+        @SerializedName("isLongClickable")
+        IS_LONG_CLICKABLE("isLongClickable"),
+
+        @SerializedName("isScrollable")
+        IS_SCROLLABLE("isScrollable"),
+
+        @SerializedName("isSelected")
+        IS_SELECTED("isSelected"),;
+
+        private final String name;
+
+        Action(final String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public static List<String> getValidActionNames() {
+            List<String> validActions = new ArrayList<>();
+            for (Action action : Action.values()) {
+                validActions.add(action.getName());
+            }
+            return validActions;
+        }
+    }
+}

lib/commands/execute.js

@@ -29,7 +29,9 @@ extensions.executeMobile = async function (mobileCommand, opts = {}) {
 
     backdoor: 'mobileBackdoor',
 
-    flashElement: 'flashElement'
+    flashElement: 'flashElement',
+
+    uiautomator: 'uiautomator'
   };
 
   if (!_.has(mobileCommandsMapping, mobileCommand)) {

lib/commands/general.js

@@ -184,6 +184,21 @@ commands.mobileBackdoor = async function (opts = {}) {
   return await this.espresso.jwproxy.command(`/appium/execute_mobile/backdoor`, 'POST', {target, methods, elementId});
 };
 
+/**
+ *  Execute UiAutomator2 commands to drive out of app areas.
+ *  strategy can be one of: "clazz", "res", "text", "textContains", "textEndsWith", "textStartsWith",
+ *                          "desc", "descContains", "descEndsWith", "descStartsWith", "pkg"
+ *
+ *  action can be one of: "click", "longClick", "getText", "getContentDescription", "getClassName",
+ *                        "getResourceName", "getVisibleBounds", "getVisibleCenter", "getApplicationPackage",
+ *                        "getChildCount", "clear", "isCheckable", "isChecked", "isClickable", "isEnabled",
+ *                        "isFocusable", "isFocused", "isLongClickable", "isScrollable", "isSelected"
+ */
+commands.uiautomator = async function (opts = {}) {
+  const {strategy, value, action, index} = assertRequiredOptions(opts, ['strategy', 'value', 'action']);
+  return await this.espresso.jwproxy.command(`/appium/execute_mobile/uiautomator`, 'POST', {strategy, value, index, action});
+};
+
 /**
  *  Flash the element with given id.
  *  durationMillis and repeatCount are optional

test/functional/commands/mobile-e2e-specs.js

@@ -93,4 +93,15 @@ describe('mobile', function () {
       await driver.execute('mobile: scrollToPage', {element: el, scrollTo: 'left', smoothScroll: true}).should.eventually.be.rejectedWith(/Could not perform scroll to on element/);
     });
   });
+
+  describe('mobile:uiautomator', function () {
+    it('should be able to find and take action on all uiObjects', async function () {
+      const text = await driver.execute('mobile: uiautomator', {strategy: 'clazz', value: 'android.widget.TextView', action: 'getText'});
+      text.should.include('Views');
+    });
+    it('should be able to find and take action on uiObject with given index', async function () {
+      const text = await driver.execute('mobile: uiautomator', {strategy: 'textContains', value: 'Views', index: 0, action: 'getText'});
+      text.should.eql(['Views']);
+    });
+  });
 });