Merge "[Thread] restarts when ot-daemon crashes" into main

thread/service/java/com/android/server/thread/ThreadNetworkControllerService.java

@@ -66,7 +66,6 @@ import android.annotation.RequiresPermission;
 import android.annotation.TargetApi;
 import android.content.Context;
 import android.net.ConnectivityManager;
-import android.net.IpPrefix;
 import android.net.LinkAddress;
 import android.net.LinkProperties;
 import android.net.LocalNetworkConfig;
@@ -79,7 +78,6 @@ import android.net.NetworkCapabilities;
 import android.net.NetworkProvider;
 import android.net.NetworkRequest;
 import android.net.NetworkScore;
-import android.net.RouteInfo;
 import android.net.TestNetworkSpecifier;
 import android.net.thread.ActiveOperationalDataset;
 import android.net.thread.ActiveOperationalDataset.SecurityPolicy;
@@ -151,7 +149,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
     private final ConnectivityManager mConnectivityManager;
     private final TunInterfaceController mTunIfController;
     private final InfraInterfaceController mInfraIfController;
-    private final LinkProperties mLinkProperties = new LinkProperties();
     private final OtDaemonCallbackProxy mOtDaemonCallbackProxy = new OtDaemonCallbackProxy();
 
     // TODO(b/308310823): read supported channel from Thread dameon
@@ -295,12 +292,16 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         return mOtDaemon;
     }
 
-    // TODO(b/309792480): restarts the OT daemon service
     private void onOtDaemonDied() {
-        Log.w(TAG, "OT daemon became dead, clean up...");
+        checkOnHandlerThread();
+        Log.w(TAG, "OT daemon is dead, clean up and restart it...");
+
         OperationReceiverWrapper.onOtDaemonDied();
         mOtDaemonCallbackProxy.onOtDaemonDied();
+        mTunIfController.onOtDaemonDied();
+
         mOtDaemon = null;
+        initializeOtDaemon();
     }
 
     public void initialize() {
@@ -313,8 +314,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
                         throw new IllegalStateException(
                                 "Failed to create Thread tunnel interface", e);
                     }
-                    mLinkProperties.setInterfaceName(TUN_IF_NAME);
-                    mLinkProperties.setMtu(TunInterfaceController.MTU);
                     mConnectivityManager.registerNetworkProvider(mNetworkProvider);
                     requestUpstreamNetwork();
                     requestThreadNetwork();
@@ -465,7 +464,7 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
                 mHandler.getLooper(),
                 TAG,
                 netCaps,
-                mLinkProperties,
+                mTunIfController.getLinkProperties(),
                 newLocalNetworkConfig(),
                 score,
                 new NetworkAgentConfig.Builder().build(),
@@ -496,46 +495,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         mNetworkAgent = null;
     }
 
-    private void updateTunInterfaceAddress(LinkAddress linkAddress, boolean isAdded) {
-        try {
-            if (isAdded) {
-                mTunIfController.addAddress(linkAddress);
-            } else {
-                mTunIfController.removeAddress(linkAddress);
-            }
-        } catch (IOException e) {
-            Log.e(
-                    TAG,
-                    String.format(
-                            "Failed to %s Thread tun interface address %s",
-                            (isAdded ? "add" : "remove"), linkAddress),
-                    e);
-        }
-    }
-
-    private void updateNetworkLinkProperties(LinkAddress linkAddress, boolean isAdded) {
-        RouteInfo routeInfo =
-                new RouteInfo(
-                        new IpPrefix(linkAddress.getAddress(), 64),
-                        null,
-                        TUN_IF_NAME,
-                        RouteInfo.RTN_UNICAST,
-                        TunInterfaceController.MTU);
-        if (isAdded) {
-            mLinkProperties.addLinkAddress(linkAddress);
-            mLinkProperties.addRoute(routeInfo);
-        } else {
-            mLinkProperties.removeLinkAddress(linkAddress);
-            mLinkProperties.removeRoute(routeInfo);
-        }
-
-        // The Thread daemon can send link property updates before the networkAgent is
-        // registered
-        if (mNetworkAgent != null) {
-            mNetworkAgent.sendLinkProperties(mLinkProperties);
-        }
-    }
-
     @Override
     public int getThreadVersion() {
         return THREAD_VERSION_1_3;
@@ -872,7 +831,7 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
     private void handleThreadInterfaceStateChanged(boolean isUp) {
         try {
             mTunIfController.setInterfaceUp(isUp);
-            Log.d(TAG, "Thread network interface becomes " + (isUp ? "up" : "down"));
+            Log.i(TAG, "Thread TUN interface becomes " + (isUp ? "up" : "down"));
         } catch (IOException e) {
             Log.e(TAG, "Failed to handle Thread interface state changes", e);
         }
@@ -880,13 +839,13 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
 
     private void handleDeviceRoleChanged(@DeviceRole int deviceRole) {
         if (ThreadNetworkController.isAttached(deviceRole)) {
-            Log.d(TAG, "Attached to the Thread network");
+            Log.i(TAG, "Attached to the Thread network");
 
             // This is an idempotent method which can be called for multiple times when the device
             // is already attached (e.g. going from Child to Router)
             registerThreadNetwork();
         } else {
-            Log.d(TAG, "Detached from the Thread network");
+            Log.i(TAG, "Detached from the Thread network");
 
             // This is an idempotent method which can be called for multiple times when the device
             // is already detached or stopped
@@ -903,10 +862,17 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         }
 
         LinkAddress linkAddress = newLinkAddress(addressInfo);
-        Log.d(TAG, (isAdded ? "Adding" : "Removing") + " address " + linkAddress);
+        if (isAdded) {
+            mTunIfController.addAddress(linkAddress);
+        } else {
+            mTunIfController.removeAddress(linkAddress);
+        }
 
-        updateTunInterfaceAddress(linkAddress, isAdded);
-        updateNetworkLinkProperties(linkAddress, isAdded);
+        // The OT daemon can send link property updates before the networkAgent is
+        // registered
+        if (mNetworkAgent != null) {
+            mNetworkAgent.sendLinkProperties(mTunIfController.getLinkProperties());
+        }
     }
 
     private boolean isMulticastForwardingEnabled() {

thread/service/java/com/android/server/thread/TunInterfaceController.java

@@ -17,7 +17,10 @@
 package com.android.server.thread;
 
 import android.annotation.Nullable;
+import android.net.IpPrefix;
 import android.net.LinkAddress;
+import android.net.LinkProperties;
+import android.net.RouteInfo;
 import android.net.util.SocketUtils;
 import android.os.ParcelFileDescriptor;
 import android.os.SystemClock;
@@ -31,6 +34,7 @@ import com.android.net.module.util.netlink.RtNetlinkAddressMessage;
 
 import java.io.FileDescriptor;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 
 /** Controller for virtual/tunnel network interfaces. */
 public class TunInterfaceController {
@@ -43,13 +47,21 @@ public class TunInterfaceController {
     }
 
     private final String mIfName;
+    private final LinkProperties mLinkProperties = new LinkProperties();
     private ParcelFileDescriptor mParcelTunFd;
     private FileDescriptor mNetlinkSocket;
     private static int sNetlinkSeqNo = 0;
 
     /** Creates a new {@link TunInterfaceController} instance for given interface. */
     public TunInterfaceController(String interfaceName) {
-        this.mIfName = interfaceName;
+        mIfName = interfaceName;
+        mLinkProperties.setInterfaceName(mIfName);
+        mLinkProperties.setMtu(MTU);
+    }
+
+    /** Returns link properties of the Thread TUN interface. */
+    public LinkProperties getLinkProperties() {
+        return mLinkProperties;
     }
 
     /**
@@ -87,13 +99,18 @@ public class TunInterfaceController {
 
     /** Sets the interface up or down according to {@code isUp}. */
     public void setInterfaceUp(boolean isUp) throws IOException {
+        if (!isUp) {
+            for (LinkAddress address : mLinkProperties.getAllLinkAddresses()) {
+                removeAddress(address);
+            }
+        }
         nativeSetInterfaceUp(mIfName, isUp);
     }
 
     private native void nativeSetInterfaceUp(String interfaceName, boolean isUp) throws IOException;
 
     /** Adds a new address to the interface. */
-    public void addAddress(LinkAddress address) throws IOException {
+    public void addAddress(LinkAddress address) {
         Log.d(TAG, "Adding address " + address + " with flags: " + address.getFlags());
 
         long validLifetimeSeconds;
@@ -121,7 +138,7 @@ public class TunInterfaceController {
 
         byte[] message =
                 RtNetlinkAddressMessage.newRtmNewAddressMessage(
-                        sNetlinkSeqNo,
+                        sNetlinkSeqNo++,
                         address.getAddress(),
                         (short) address.getPrefixLength(),
                         address.getFlags(),
@@ -131,13 +148,51 @@ public class TunInterfaceController {
                         preferredLifetimeSeconds);
         try {
             Os.write(mNetlinkSocket, message, 0, message.length);
-        } catch (ErrnoException e) {
-            throw new IOException("Failed to send netlink message", e);
+        } catch (ErrnoException | InterruptedIOException e) {
+            Log.e(TAG, "Failed to add address " + address, e);
+            return;
         }
+        mLinkProperties.addLinkAddress(address);
+        mLinkProperties.addRoute(getRouteForAddress(address));
     }
 
     /** Removes an address from the interface. */
-    public void removeAddress(LinkAddress address) throws IOException {
-        // TODO(b/263222068): remove address with netlink
+    public void removeAddress(LinkAddress address) {
+        Log.d(TAG, "Removing address " + address);
+        byte[] message =
+                RtNetlinkAddressMessage.newRtmDelAddressMessage(
+                        sNetlinkSeqNo++,
+                        address.getAddress(),
+                        (short) address.getPrefixLength(),
+                        Os.if_nametoindex(mIfName));
+
+        // Intentionally update the mLinkProperties before send netlink message because the
+        // address is already removed from ot-daemon and apps can't reach to the address even
+        // when the netlink request below fails
+        mLinkProperties.removeLinkAddress(address);
+        mLinkProperties.removeRoute(getRouteForAddress(address));
+        try {
+            Os.write(mNetlinkSocket, message, 0, message.length);
+        } catch (ErrnoException | InterruptedIOException e) {
+            Log.e(TAG, "Failed to remove address " + address, e);
+        }
+    }
+
+    private RouteInfo getRouteForAddress(LinkAddress linkAddress) {
+        return new RouteInfo(
+                new IpPrefix(linkAddress.getAddress(), linkAddress.getPrefixLength()),
+                null,
+                mIfName,
+                RouteInfo.RTN_UNICAST,
+                MTU);
+    }
+
+    /** Called by {@link ThreadNetworkControllerService} to do clean up when ot-daemon is dead. */
+    public void onOtDaemonDied() {
+        try {
+            setInterfaceUp(false);
+        } catch (IOException e) {
+            Log.e(TAG, "Failed to set Thread TUN interface down");
+        }
     }
 }

thread/tests/integration/Android.bp

@@ -24,12 +24,14 @@ java_defaults {
     min_sdk_version: "30",
     static_libs: [
         "androidx.test.rules",
+        "compatibility-device-util-axt",
         "guava",
         "mockito-target-minus-junit4",
         "net-tests-utils",
         "net-utils-device-common",
         "net-utils-device-common-bpf",
         "testables",
+        "truth",
     ],
     libs: [
         "android.test.runner",

thread/tests/integration/AndroidTest.xml

@@ -31,6 +31,8 @@
         <option name="mainline-module-package-name" value="com.google.android.tethering" />
     </object>
 
+    <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer" />
+
     <!-- Install test -->
     <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
         <option name="test-file-name" value="ThreadNetworkIntegrationTests.apk" />

thread/tests/integration/src/android/net/thread/BorderRoutingTest.java

@@ -18,14 +18,15 @@ package android.net.thread;
 
 import static android.Manifest.permission.MANAGE_TEST_NETWORKS;
 import static android.Manifest.permission.NETWORK_SETTINGS;
-import static android.net.thread.IntegrationTestUtils.isExpectedIcmpv6Packet;
-import static android.net.thread.IntegrationTestUtils.isSimulatedThreadRadioSupported;
-import static android.net.thread.IntegrationTestUtils.newPacketReader;
-import static android.net.thread.IntegrationTestUtils.readPacketFrom;
-import static android.net.thread.IntegrationTestUtils.waitFor;
-import static android.net.thread.IntegrationTestUtils.waitForStateAnyOf;
 import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_LEADER;
 import static android.net.thread.ThreadNetworkManager.PERMISSION_THREAD_NETWORK_PRIVILEGED;
+import static android.net.thread.utils.IntegrationTestUtils.JOIN_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.isExpectedIcmpv6Packet;
+import static android.net.thread.utils.IntegrationTestUtils.isSimulatedThreadRadioSupported;
+import static android.net.thread.utils.IntegrationTestUtils.newPacketReader;
+import static android.net.thread.utils.IntegrationTestUtils.readPacketFrom;
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
+import static android.net.thread.utils.IntegrationTestUtils.waitForStateAnyOf;
 
 import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ECHO_REPLY_TYPE;
 import static com.android.testutils.TestNetworkTrackerKt.initTestNetwork;
@@ -41,6 +42,8 @@ import static org.junit.Assume.assumeTrue;
 import android.content.Context;
 import android.net.LinkProperties;
 import android.net.MacAddress;
+import android.net.thread.utils.FullThreadDevice;
+import android.net.thread.utils.InfraNetworkDevice;
 import android.os.Handler;
 import android.os.HandlerThread;
 
@@ -57,6 +60,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import java.net.Inet6Address;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -155,14 +159,14 @@ public class BorderRoutingTest {
         runAsShell(
                 PERMISSION_THREAD_NETWORK_PRIVILEGED,
                 () -> mController.join(DEFAULT_DATASET, directExecutor(), result -> {}));
-        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), 30 /* timeoutSeconds */);
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), JOIN_TIMEOUT);
 
         // Creates a Full Thread Device (FTD) and lets it join the network.
         FullThreadDevice ftd = new FullThreadDevice(5 /* node ID */);
         ftd.factoryReset();
         ftd.joinNetwork(DEFAULT_DATASET);
-        ftd.waitForStateAnyOf(List.of("router", "child"), 10 /* timeoutSeconds */);
-        waitFor(() -> ftd.getOmrAddress() != null, 60 /* timeoutSeconds */);
+        ftd.waitForStateAnyOf(List.of("router", "child"), JOIN_TIMEOUT);
+        waitFor(() -> ftd.getOmrAddress() != null, Duration.ofSeconds(60));
         Inet6Address ftdOmr = ftd.getOmrAddress();
         assertNotNull(ftdOmr);
 
@@ -171,7 +175,7 @@ public class BorderRoutingTest {
                 newPacketReader(mInfraNetworkTracker.getTestIface(), mHandler);
         InfraNetworkDevice infraDevice =
                 new InfraNetworkDevice(MacAddress.fromString("1:2:3:4:5:6"), infraNetworkReader);
-        infraDevice.runSlaac(60 /* timeoutSeconds */);
+        infraDevice.runSlaac(Duration.ofSeconds(60));
         assertNotNull(infraDevice.ipv6Addr);
 
         // Infra device sends an echo request to FTD's OMR.

thread/tests/integration/src/android/net/thread/ThreadIntegrationTest.java

+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net.thread;
+
+import static android.Manifest.permission.NETWORK_SETTINGS;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_DETACHED;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_LEADER;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_STOPPED;
+import static android.net.thread.ThreadNetworkManager.PERMISSION_THREAD_NETWORK_PRIVILEGED;
+import static android.net.thread.utils.IntegrationTestUtils.CALLBACK_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.LEAVE_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.RESTART_JOIN_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.waitForStateAnyOf;
+
+import static com.android.compatibility.common.util.SystemUtil.runShellCommand;
+import static com.android.testutils.TestPermissionUtil.runAsShell;
+
+import static com.google.common.io.BaseEncoding.base16;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+
+import static org.junit.Assume.assumeNotNull;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import android.annotation.Nullable;
+import android.content.Context;
+import android.net.thread.ThreadNetworkController.StateCallback;
+import android.net.thread.utils.OtDaemonController;
+import android.os.SystemClock;
+
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.filters.LargeTest;
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.net.Inet6Address;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+/** Tests for E2E Android Thread integration with ot-daemon, ConnectivityService, etc.. */
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class ThreadIntegrationTest {
+    private final Context mContext = ApplicationProvider.getApplicationContext();
+    private ThreadNetworkController mController;
+    private OtDaemonController mOtCtl;
+
+    // A valid Thread Active Operational Dataset generated from OpenThread CLI "dataset init new".
+    private static final byte[] DEFAULT_DATASET_TLVS =
+            base16().decode(
+                            "0E080000000000010000000300001335060004001FFFE002"
+                                    + "08ACC214689BC40BDF0708FD64DB1225F47E0B0510F26B31"
+                                    + "53760F519A63BAFDDFFC80D2AF030F4F70656E5468726561"
+                                    + "642D643961300102D9A00410A245479C836D551B9CA557F7"
+                                    + "B9D351B40C0402A0FFF8");
+    private static final ActiveOperationalDataset DEFAULT_DATASET =
+            ActiveOperationalDataset.fromThreadTlvs(DEFAULT_DATASET_TLVS);
+
+    @Before
+    public void setUp() throws Exception {
+        final ThreadNetworkManager manager = mContext.getSystemService(ThreadNetworkManager.class);
+        if (manager != null) {
+            mController = manager.getAllThreadNetworkControllers().get(0);
+        }
+
+        // Run the tests on only devices where the Thread feature is available
+        assumeNotNull(mController);
+
+        mOtCtl = new OtDaemonController();
+        leaveAndWait(mController);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (mController == null) {
+            return;
+        }
+
+        setTestUpStreamNetworkAndWait(mController, null);
+        leaveAndWait(mController);
+    }
+
+    @Test
+    public void otDaemonRestart_notJoinedAndStopped_deviceRoleIsStopped() throws Exception {
+        leaveAndWait(mController);
+
+        runShellCommand("stop ot-daemon");
+        // TODO(b/323331973): the sleep is needed to workaround the race conditions
+        SystemClock.sleep(200);
+
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_STOPPED), CALLBACK_TIMEOUT);
+    }
+
+    @Test
+    public void otDaemonRestart_JoinedNetworkAndStopped_autoRejoined() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        runShellCommand("stop ot-daemon");
+
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_DETACHED), CALLBACK_TIMEOUT);
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), RESTART_JOIN_TIMEOUT);
+    }
+
+    @Test
+    public void otDaemonFactoryReset_deviceRoleIsStopped() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        mOtCtl.factoryReset();
+
+        assertThat(getDeviceRole(mController)).isEqualTo(DEVICE_ROLE_STOPPED);
+    }
+
+    @Test
+    public void otDaemonFactoryReset_addressesRemoved() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        mOtCtl.factoryReset();
+        String ifconfig = runShellCommand("ifconfig thread-wpan");
+
+        assertThat(ifconfig).doesNotContain("inet6 addr");
+    }
+
+    @Test
+    public void tunInterface_joinedNetwork_otAddressesAddedToTunInterface() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        String ifconfig = runShellCommand("ifconfig thread-wpan");
+        List<Inet6Address> otAddresses = mOtCtl.getAddresses();
+        assertThat(otAddresses).isNotEmpty();
+        for (Inet6Address otAddress : otAddresses) {
+            assertThat(ifconfig).contains(otAddress.getHostAddress());
+        }
+    }
+
+    // TODO (b/323300829): add more tests for integration with linux platform and
+    // ConnectivityService
+
+    private static int getDeviceRole(ThreadNetworkController controller) throws Exception {
+        CompletableFuture<Integer> future = new CompletableFuture<>();
+        StateCallback callback = future::complete;
+        controller.registerStateCallback(directExecutor(), callback);
+        try {
+            return future.get(CALLBACK_TIMEOUT.toMillis(), MILLISECONDS);
+        } finally {
+            controller.unregisterStateCallback(callback);
+        }
+    }
+
+    private static void joinAndWait(
+            ThreadNetworkController controller, ActiveOperationalDataset activeDataset)
+            throws Exception {
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                () -> controller.join(activeDataset, directExecutor(), result -> {}));
+        waitForStateAnyOf(controller, List.of(DEVICE_ROLE_LEADER), RESTART_JOIN_TIMEOUT);
+    }
+
+    private static void leaveAndWait(ThreadNetworkController controller) throws Exception {
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                () -> controller.leave(directExecutor(), future::complete));
+        future.get(LEAVE_TIMEOUT.toMillis(), MILLISECONDS);
+    }
+
+    private static void setTestUpStreamNetworkAndWait(
+            ThreadNetworkController controller, @Nullable String networkInterfaceName)
+            throws Exception {
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                NETWORK_SETTINGS,
+                () -> {
+                    controller.setTestNetworkAsUpstream(
+                            networkInterfaceName, directExecutor(), future::complete);
+                });
+        future.get(CALLBACK_TIMEOUT.toMillis(), MILLISECONDS);
+    }
+}

thread/tests/integration/src/android/net/thread/FullThreadDevice.java → thread/tests/integration/src/android/net/thread/utils/FullThreadDevice.java

@@ -13,9 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package android.net.thread;
+package android.net.thread.utils;
 
-import static android.net.thread.IntegrationTestUtils.waitFor;
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
 
 import static com.google.common.io.BaseEncoding.base16;
 
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
 
 import android.net.InetAddresses;
 import android.net.IpPrefix;
+import android.net.thread.ActiveOperationalDataset;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -30,6 +31,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
 import java.net.Inet6Address;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
@@ -115,10 +117,10 @@ public final class FullThreadDevice {
      *
      * @param states the list of states to wait for. Valid states are "disabled", "detached",
      *     "child", "router" and "leader".
-     * @param timeoutSeconds the number of seconds to wait for.
+     * @param timeout the time to wait for the expected state before throwing
      */
-    public void waitForStateAnyOf(List<String> states, int timeoutSeconds) throws TimeoutException {
-        waitFor(() -> states.contains(getState()), timeoutSeconds);
+    public void waitForStateAnyOf(List<String> states, Duration timeout) throws TimeoutException {
+        waitFor(() -> states.contains(getState()), timeout);
     }
 
     /**

thread/tests/integration/src/android/net/thread/InfraNetworkDevice.java → thread/tests/integration/src/android/net/thread/utils/InfraNetworkDevice.java

@@ -13,11 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package android.net.thread;
+package android.net.thread.utils;
 
-import static android.net.thread.IntegrationTestUtils.getRaPios;
-import static android.net.thread.IntegrationTestUtils.readPacketFrom;
-import static android.net.thread.IntegrationTestUtils.waitFor;
+import static android.net.thread.utils.IntegrationTestUtils.getRaPios;
+import static android.net.thread.utils.IntegrationTestUtils.readPacketFrom;
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
 
 import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ND_OPTION_SLLA;
 import static com.android.net.module.util.NetworkStackConstants.IPV6_ADDR_ALL_ROUTERS_MULTICAST;
@@ -34,6 +34,7 @@ import java.io.IOException;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.nio.ByteBuffer;
+import java.time.Duration;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.TimeoutException;
@@ -100,8 +101,8 @@ public final class InfraNetworkDevice {
      * @param timeoutSeconds the number of seconds to wait for.
      * @throws TimeoutException when the device fails to generate a SLAAC address in given timeout.
      */
-    public void runSlaac(int timeoutSeconds) throws TimeoutException {
-        waitFor(() -> (ipv6Addr = runSlaac()) != null, timeoutSeconds, 5 /* intervalSeconds */);
+    public void runSlaac(Duration timeout) throws TimeoutException {
+        waitFor(() -> (ipv6Addr = runSlaac()) != null, timeout);
     }
 
     private Inet6Address runSlaac() {

thread/tests/integration/src/android/net/thread/IntegrationTestUtils.java → thread/tests/integration/src/android/net/thread/utils/IntegrationTestUtils.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package android.net.thread;
+package android.net.thread.utils;
 
 import static android.system.OsConstants.IPPROTO_ICMPV6;
 
@@ -23,6 +23,7 @@ import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ROUTER_AD
 import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
 
 import android.net.TestNetworkInterface;
+import android.net.thread.ThreadNetworkController;
 import android.os.Handler;
 import android.os.SystemClock;
 import android.os.SystemProperties;
@@ -39,6 +40,7 @@ import com.google.common.util.concurrent.SettableFuture;
 
 import java.io.FileDescriptor;
 import java.nio.ByteBuffer;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -49,6 +51,14 @@ import java.util.function.Supplier;
 
 /** Static utility methods relating to Thread integration tests. */
 public final class IntegrationTestUtils {
+    // The timeout of join() after restarting ot-daemon. The device needs to send 6 Link Request
+    // every 5 seconds, followed by 4 Parent Request every second. So this value needs to be 40
+    // seconds to be safe
+    public static final Duration RESTART_JOIN_TIMEOUT = Duration.ofSeconds(40);
+    public static final Duration JOIN_TIMEOUT = Duration.ofSeconds(30);
+    public static final Duration LEAVE_TIMEOUT = Duration.ofSeconds(2);
+    public static final Duration CALLBACK_TIMEOUT = Duration.ofSeconds(1);
+
     private IntegrationTestUtils() {}
 
     /** Returns whether the device supports simulated Thread radio. */
@@ -60,49 +70,33 @@ public final class IntegrationTestUtils {
     /**
      * Waits for the given {@link Supplier} to be true until given timeout.
      *
-     * <p>It checks the condition once every second.
-     *
-     * @param condition the condition to check.
-     * @param timeoutSeconds the number of seconds to wait for.
-     * @throws TimeoutException if the condition is not met after the timeout.
+     * @param condition the condition to check
+     * @param timeout the time to wait for the condition before throwing
+     * @throws TimeoutException if the condition is still not met when the timeout expires
      */
-    public static void waitFor(Supplier<Boolean> condition, int timeoutSeconds)
+    public static void waitFor(Supplier<Boolean> condition, Duration timeout)
             throws TimeoutException {
-        waitFor(condition, timeoutSeconds, 1);
-    }
+        final long intervalMills = 1000;
+        final long timeoutMills = timeout.toMillis();
 
-    /**
-     * Waits for the given {@link Supplier} to be true until given timeout.
-     *
-     * <p>It checks the condition once every {@code intervalSeconds}.
-     *
-     * @param condition the condition to check.
-     * @param timeoutSeconds the number of seconds to wait for.
-     * @param intervalSeconds the period to check the {@code condition}.
-     * @throws TimeoutException if the condition is still not met when the timeout expires.
-     */
-    public static void waitFor(Supplier<Boolean> condition, int timeoutSeconds, int intervalSeconds)
-            throws TimeoutException {
-        for (int i = 0; i < timeoutSeconds; i += intervalSeconds) {
+        for (long i = 0; i < timeoutMills; i += intervalMills) {
             if (condition.get()) {
                 return;
             }
-            SystemClock.sleep(intervalSeconds * 1000L);
+            SystemClock.sleep(intervalMills);
         }
         if (condition.get()) {
             return;
         }
-        throw new TimeoutException(
-                String.format(
-                        "The condition failed to become true in %d seconds.", timeoutSeconds));
+        throw new TimeoutException("The condition failed to become true in " + timeout);
     }
 
     /**
      * Creates a {@link TapPacketReader} given the {@link TestNetworkInterface} and {@link Handler}.
      *
-     * @param testNetworkInterface the TUN interface of the test network.
-     * @param handler the handler to process the packets.
-     * @return the {@link TapPacketReader}.
+     * @param testNetworkInterface the TUN interface of the test network
+     * @param handler the handler to process the packets
+     * @return the {@link TapPacketReader}
      */
     public static TapPacketReader newPacketReader(
             TestNetworkInterface testNetworkInterface, Handler handler) {
@@ -117,16 +111,16 @@ public final class IntegrationTestUtils {
     /**
      * Waits for the Thread module to enter any state of the given {@code deviceRoles}.
      *
-     * @param controller the {@link ThreadNetworkController}.
+     * @param controller the {@link ThreadNetworkController}
      * @param deviceRoles the desired device roles. See also {@link
-     *     ThreadNetworkController.DeviceRole}.
-     * @param timeoutSeconds the number of seconds ot wait for.
-     * @return the {@link ThreadNetworkController.DeviceRole} after waiting.
+     *     ThreadNetworkController.DeviceRole}
+     * @param timeout the time to wait for the expected state before throwing
+     * @return the {@link ThreadNetworkController.DeviceRole} after waiting
      * @throws TimeoutException if the device hasn't become any of expected roles until the timeout
-     *     expires.
+     *     expires
      */
     public static int waitForStateAnyOf(
-            ThreadNetworkController controller, List<Integer> deviceRoles, int timeoutSeconds)
+            ThreadNetworkController controller, List<Integer> deviceRoles, Duration timeout)
             throws TimeoutException {
         SettableFuture<Integer> future = SettableFuture.create();
         ThreadNetworkController.StateCallback callback =
@@ -137,24 +131,24 @@ public final class IntegrationTestUtils {
                 };
         controller.registerStateCallback(directExecutor(), callback);
         try {
-            int role = future.get(timeoutSeconds, TimeUnit.SECONDS);
-            controller.unregisterStateCallback(callback);
-            return role;
+            return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
         } catch (InterruptedException | ExecutionException e) {
             throw new TimeoutException(
                     String.format(
-                            "The device didn't become an expected role in %d seconds.",
-                            timeoutSeconds));
+                            "The device didn't become an expected role in %s: %s",
+                            timeout, e.getMessage()));
+        } finally {
+            controller.unregisterStateCallback(callback);
         }
     }
 
     /**
      * Reads a packet from a given {@link TapPacketReader} that satisfies the {@code filter}.
      *
-     * @param packetReader a TUN packet reader.
-     * @param filter the filter to be applied on the packet.
+     * @param packetReader a TUN packet reader
+     * @param filter the filter to be applied on the packet
      * @return the first IPv6 packet that satisfies the {@code filter}. If it has waited for more
-     *     than 3000ms to read the next packet, the method will return null.
+     *     than 3000ms to read the next packet, the method will return null
      */
     public static byte[] readPacketFrom(TapPacketReader packetReader, Predicate<byte[]> filter) {
         byte[] packet;

thread/tests/integration/src/android/net/thread/utils/OtDaemonController.java

+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net.thread.utils;
+
+import android.net.InetAddresses;
+import android.os.SystemClock;
+
+import com.android.compatibility.common.util.SystemUtil;
+
+import java.net.Inet6Address;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Wrapper of the "/system/bin/ot-ctl" which can be used to send CLI commands to ot-daemon to
+ * control its behavior.
+ *
+ * <p>Note that this class takes root privileged to run.
+ */
+public final class OtDaemonController {
+    private static final String OT_CTL = "/system/bin/ot-ctl";
+
+    /**
+     * Factory resets ot-daemon.
+     *
+     * <p>This will erase all persistent data written into apexdata/com.android.apex/ot-daemon and
+     * restart the ot-daemon service.
+     */
+    public void factoryReset() {
+        executeCommand("factoryreset");
+
+        // TODO(b/323164524): ot-ctl is a separate process so that the tests can't depend on the
+        // time sequence. Here needs to wait for system server to receive the ot-daemon death
+        // signal and take actions.
+        // A proper fix is to replace "ot-ctl" with "cmd thread_network ot-ctl" which is
+        // synchronized with the system server
+        SystemClock.sleep(500);
+    }
+
+    /** Returns the list of IPv6 addresses on ot-daemon. */
+    public List<Inet6Address> getAddresses() {
+        String output = executeCommand("ipaddr");
+        return Arrays.asList(output.split("\n")).stream()
+                .map(String::trim)
+                .filter(str -> !str.equals("Done"))
+                .map(addr -> InetAddresses.parseNumericAddress(addr))
+                .map(inetAddr -> (Inet6Address) inetAddr)
+                .toList();
+    }
+
+    public String executeCommand(String cmd) {
+        return SystemUtil.runShellCommand(OT_CTL + " " + cmd);
+    }
+}