floss: Add hcidoc tool to read snoop files

Hcidoc will be used to analyze snoop files and snooz files to identify
errors and other behaviors. This commit simply sets up the hcidoc crate
and a simple snoop reader.

Bug: 262928525
Tag: #floss
Test: ./build.py --target utils
Change-Id: I7eeae136a35d51a01ac2ab137dbf114e07f800e9

Cargo.toml

@@ -15,10 +15,19 @@
 
 [workspace]
 
+default-members = [
+  "system/gd/rust/shim",
+  "system/gd/rust/topshim",
+  "system/gd/rust/linux/mgmt",
+  "system/gd/rust/linux/service",
+  "system/gd/rust/linux/client",
+]
+
 members = [
   "system/gd/rust/shim",
   "system/gd/rust/topshim",
   "system/gd/rust/linux/mgmt",
   "system/gd/rust/linux/service",
   "system/gd/rust/linux/client",
+  "floss/hcidoc",
 ]

build.py

@@ -67,12 +67,13 @@ VALID_TARGETS = [
     'all',  # All targets except test and clean
     'clean',  # Clean up output directory
     'docs',  # Build Rust docs
+    'hosttools',  # Build the host tools (i.e. packetgen)
     'main',  # Build the main C++ codebase
     'prepare',  # Prepare the output directory (gn gen + rust setup)
     'rootcanal',  # Build Rust targets for RootCanal
     'rust',  # Build only the rust components + copy artifacts to output dir
     'test',  # Run the unit tests
-    'tools',  # Build the host tools (i.e. packetgen)
+    'utils',  # Build Floss utils
 ]
 
 # TODO(b/190750167) - Host tests are disabled until we are full bazel build
@@ -440,14 +441,14 @@ class HostBuild():
         self._gn_configure()
         self._rust_configure()
 
-    def _target_tools(self):
+    def _target_hosttools(self):
         """ Build the tools target in an already prepared environment.
         """
         self._gn_build('tools')
 
         # Also copy bluetooth_packetgen to CARGO_HOME so it's available
-        shutil.copy(
-            os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), os.path.join(self.env['CARGO_HOME'], 'bin'))
+        shutil.copy(os.path.join(self._gn_default_output(), 'bluetooth_packetgen'),
+                    os.path.join(self.env['CARGO_HOME'], 'bin'))
 
     def _target_docs(self):
         """Build the Rust docs."""
@@ -488,10 +489,20 @@ class HostBuild():
 
         # Host tests second based on host test list
         for t in HOST_TESTS:
-            self.run_command(
-                'test', [os.path.join(self.output_dir, 'out/Default', t)],
-                cwd=os.path.join(self.output_dir),
-                env=self.env)
+            self.run_command('test', [os.path.join(self.output_dir, 'out/Default', t)],
+                             cwd=os.path.join(self.output_dir),
+                             env=self.env)
+
+    def _target_utils(self):
+        """ Builds the utility applications.
+        """
+        rust_targets = ['hcidoc']
+
+        # Build targets
+        for target in rust_targets:
+            self.run_command('utils', ['cargo', 'build', '-p', target],
+                             cwd=os.path.join(self.platform_dir, 'bt'),
+                             env=self.env)
 
     def _target_install(self):
         """ Installs files required to run Floss to install directory.
@@ -568,7 +579,7 @@ class HostBuild():
         """ Build all common targets (skipping doc, test, and clean).
         """
         self._target_prepare()
-        self._target_tools()
+        self._target_hosttools()
         self._target_main()
         self._target_rust()
 
@@ -584,8 +595,8 @@ class HostBuild():
 
         if self.target == 'prepare':
             self._target_prepare()
-        elif self.target == 'tools':
-            self._target_tools()
+        elif self.target == 'hosttools':
+            self._target_hosttools()
         elif self.target == 'rootcanal':
             self._target_rootcanal()
         elif self.target == 'rust':
@@ -600,6 +611,8 @@ class HostBuild():
             self._target_clean()
         elif self.target == 'install':
             self._target_install()
+        elif self.target == 'utils':
+            self._target_utils()
         elif self.target == 'all':
             self._target_all()
 
@@ -846,18 +859,22 @@ class Bootstrap():
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Simple build for host.')
-    parser.add_argument(
-        '--bootstrap-dir', help='Directory to run bootstrap on (or was previously run on).', default="~/.floss")
-    parser.add_argument(
-        '--run-bootstrap',
-        help='Run bootstrap code to verify build env is ok to build.',
-        default=False,
-        action='store_true')
-    parser.add_argument(
-        '--print-env', help='Print environment variables used for build.', default=False, action='store_true')
+    parser.add_argument('--bootstrap-dir',
+                        help='Directory to run bootstrap on (or was previously run on).',
+                        default="~/.floss")
+    parser.add_argument('--run-bootstrap',
+                        help='Run bootstrap code to verify build env is ok to build.',
+                        default=False,
+                        action='store_true')
+    parser.add_argument('--print-env',
+                        help='Print environment variables used for build.',
+                        default=False,
+                        action='store_true')
     parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true')
-    parser.add_argument(
-        '--no-strip', help='Skip stripping binaries during install.', default=False, action='store_true')
+    parser.add_argument('--no-strip',
+                        help='Skip stripping binaries during install.',
+                        default=False,
+                        action='store_true')
     parser.add_argument('--use', help='Set a specific use flag.')
     parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true')
     parser.add_argument('--test-name', help='Run test with this string in the name.', default=None)
@@ -865,8 +882,10 @@ if __name__ == '__main__':
     parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/')
     parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib')
     parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int)
-    parser.add_argument(
-        '--no-vendored-rust', help='Do not use vendored rust crates', default=False, action='store_true')
+    parser.add_argument('--no-vendored-rust',
+                        help='Do not use vendored rust crates',
+                        default=False,
+                        action='store_true')
     parser.add_argument('--verbose', help='Verbose logs for build.')
     parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true')
     parser.add_argument(

floss/hcidoc/Cargo.toml

+[package]
+name = "hcidoc"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+bt_packets = { path = "../../system/gd/rust/packets" }
+clap = "4.0"
+num-derive = "0.3"
+num-traits = "0.2"
+tokio = { version = "1.0", features = ['bytes', 'fs', 'io-util', 'libc', 'macros', 'memchr', 'mio', 'net', 'num_cpus', 'rt', 'rt-multi-thread', 'sync', 'time', 'tokio-macros'] }

floss/hcidoc/src/main.rs

+#[macro_use]
+extern crate num_derive;
+
+mod parser;
+
+use crate::parser::{LogParser, LogType};
+use bt_packets;
+use clap::{Arg, Command};
+
+fn main() {
+    let matches = Command::new("hcidoc")
+        .version("0.1")
+        .author("Abhishek Pandit-Subedi <abhishekpandit@google.com>")
+        .about("Analyzes a linux HCI snoop log for specific behaviors and errors.")
+        .arg(Arg::new("filename"))
+        .get_matches();
+
+    let filename = match matches.get_one::<String>("filename") {
+        Some(f) => f,
+        None => {
+            println!("No filename parameter given.");
+            return;
+        }
+    };
+
+    let mut parser = match LogParser::new(filename.as_str()) {
+        Ok(p) => p,
+        Err(e) => {
+            println!("Failed to load parser on {}: {}", filename, e);
+            return;
+        }
+    };
+
+    let log_type = match parser.read_log_type() {
+        Ok(v) => v,
+        Err(e) => {
+            println!("Parsing {} failed: {}", filename, e);
+            return;
+        }
+    };
+
+    if let LogType::LinuxSnoop(header) = log_type {
+        println!("Reading snoop file: {:?}", header);
+        for (pos, v) in parser.get_snoop_iterator().expect("Not a linux snoop file").enumerate() {
+            println!("#{} - Packet= {:?}, Index={}, Opcode={:?}", pos, v, v.index(), v.opcode());
+            if pos > 100 {
+                return;
+            }
+        }
+    }
+}

floss/hcidoc/src/parser.rs

+//! Parsing of various Bluetooth packets.
+use num_traits::cast::{FromPrimitive, ToPrimitive};
+use std::convert::TryFrom;
+use std::fs::File;
+use std::io::{Error, ErrorKind, Read, Seek};
+
+/// Linux snoop file header format. This format is used by `btmon` on Linux systems that have bluez
+/// installed.
+#[derive(Clone, Copy, Debug)]
+pub struct LinuxSnoopHeader {
+    id: [u8; 8],
+    version: u32,
+    data_type: u32,
+}
+
+/// Identifier for a Linux snoop file. In ASCII, this is 'btsnoop\0'.
+const LINUX_SNOOP_MAGIC: [u8; 8] = [0x62, 0x74, 0x73, 0x6e, 0x6f, 0x6f, 0x70, 0x00];
+
+/// Snoop files in monitor format will have this value in link type.
+const LINUX_SNOOP_MONITOR_TYPE: u32 = 2001;
+
+/// Size of snoop header. 8 bytes for magic and another 8 for additional info.
+const LINUX_SNOOP_HEADER_SIZE: usize = 16;
+
+impl TryFrom<&[u8]> for LinuxSnoopHeader {
+    type Error = String;
+
+    fn try_from(item: &[u8]) -> Result<Self, Self::Error> {
+        if item.len() != LINUX_SNOOP_HEADER_SIZE {
+            return Err(format!("Invalid size for snoop header: {}", item.len()));
+        }
+
+        let rest = item;
+        let (id_bytes, rest) = rest.split_at(8);
+        let (version_bytes, rest) = rest.split_at(std::mem::size_of::<u32>());
+        let (data_type_bytes, _rest) = rest.split_at(std::mem::size_of::<u32>());
+
+        let header = LinuxSnoopHeader {
+            id: id_bytes.try_into().unwrap(),
+            version: u32::from_be_bytes(version_bytes.try_into().unwrap()),
+            data_type: u32::from_be_bytes(data_type_bytes.try_into().unwrap()),
+        };
+
+        if header.id != LINUX_SNOOP_MAGIC {
+            return Err(format!("Id is not 'btsnoop'."));
+        }
+
+        if header.version != 1 {
+            return Err(format!("Version is not supported. Got {}.", header.version));
+        }
+
+        if header.data_type != LINUX_SNOOP_MONITOR_TYPE {
+            return Err(format!(
+                "Invalid data type in snoop file. We want monitor type ({}) but got {}",
+                LINUX_SNOOP_MONITOR_TYPE, header.data_type
+            ));
+        }
+
+        Ok(header)
+    }
+}
+
+/// Opcodes for Linux snoop packets.
+#[derive(Debug, FromPrimitive, ToPrimitive)]
+#[repr(u16)]
+pub enum LinuxSnoopOpcodes {
+    NewIndex = 0,
+    DeleteIndex,
+    CommandPacket,
+    EventPacket,
+    AclTxPacket,
+    AclRxPacket,
+    ScoTxPacket,
+    ScoRxPacket,
+    OpenIndex,
+    CloseIndex,
+    IndexInfo,
+    VendorDiag,
+    SystemNote,
+    UserLogging,
+    CtrlOpen,
+    CtrlClose,
+    CtrlCommand,
+    CtrlEvent,
+    IsoTx,
+    IsoRx,
+
+    Invalid = 0xffff,
+}
+
+/// Linux snoop file packet format.
+#[derive(Debug, Clone)]
+pub struct LinuxSnoopPacket {
+    /// The original length of the captured packet as received via a network.
+    pub original_length: u32,
+
+    /// The length of the included data (can be smaller than original_length if
+    /// the received packet was truncated).
+    pub included_length: u32,
+    pub flags: u32,
+    pub drops: u32,
+    pub timestamp_ms: u64,
+    pub data: Vec<u8>,
+}
+
+impl LinuxSnoopPacket {
+    pub fn index(&self) -> u16 {
+        (self.flags >> 16).try_into().unwrap_or(0u16)
+    }
+
+    pub fn opcode(&self) -> LinuxSnoopOpcodes {
+        LinuxSnoopOpcodes::from_u32(self.flags & 0xffff).unwrap_or(LinuxSnoopOpcodes::Invalid)
+    }
+}
+
+/// Size of packet preamble (everything except the data).
+const LINUX_SNOOP_PACKET_PREAMBLE_SIZE: usize = 24;
+
+/// Maximum packet size for snoop is the max ACL size + 4 bytes.
+const LINUX_SNOOP_MAX_PACKET_SIZE: usize = 1486 + 4;
+
+// Expect specifically the pre-amble to be read here (and no data).
+impl TryFrom<&[u8]> for LinuxSnoopPacket {
+    type Error = String;
+
+    fn try_from(item: &[u8]) -> Result<Self, Self::Error> {
+        if item.len() != LINUX_SNOOP_PACKET_PREAMBLE_SIZE {
+            return Err(format!("Wrong size for snoop packet preamble: {}", item.len()));
+        }
+
+        let rest = item;
+        let (orig_len_bytes, rest) = rest.split_at(std::mem::size_of::<u32>());
+        let (included_len_bytes, rest) = rest.split_at(std::mem::size_of::<u32>());
+        let (flags_bytes, rest) = rest.split_at(std::mem::size_of::<u32>());
+        let (drops_bytes, rest) = rest.split_at(std::mem::size_of::<u32>());
+        let (ts_bytes, _rest) = rest.split_at(std::mem::size_of::<u64>());
+
+        // Note that all bytes are in big-endian because they're network order.
+        let packet = LinuxSnoopPacket {
+            original_length: u32::from_be_bytes(orig_len_bytes.try_into().unwrap()),
+            included_length: u32::from_be_bytes(included_len_bytes.try_into().unwrap()),
+            flags: u32::from_be_bytes(flags_bytes.try_into().unwrap()),
+            drops: u32::from_be_bytes(drops_bytes.try_into().unwrap()),
+            timestamp_ms: u64::from_be_bytes(ts_bytes.try_into().unwrap()),
+            data: vec![],
+        };
+
+        Ok(packet)
+    }
+}
+
+/// Reader for Linux snoop files.
+pub struct LinuxSnoopReader<'a> {
+    fd: &'a File,
+}
+
+impl<'a> LinuxSnoopReader<'a> {
+    fn new(fd: &'a File) -> Self {
+        LinuxSnoopReader { fd }
+    }
+}
+
+impl<'a> Iterator for LinuxSnoopReader<'a> {
+    type Item = LinuxSnoopPacket;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut data = [0u8; LINUX_SNOOP_PACKET_PREAMBLE_SIZE];
+        let bytes = match self.fd.read(&mut data) {
+            Ok(b) => b,
+            Err(e) => {
+                // |UnexpectedEof| could be seen since we're trying to read more
+                // data than is available (i.e. end of file).
+                if e.kind() != ErrorKind::UnexpectedEof {
+                    println!("Error reading snoop file: {:?}", e);
+                }
+                return None;
+            }
+        };
+
+        match LinuxSnoopPacket::try_from(&data[0..bytes]) {
+            Ok(mut p) => {
+                if p.included_length > 0 {
+                    let size: usize = p.included_length.try_into().unwrap();
+                    let mut rem_data = [0u8; LINUX_SNOOP_MAX_PACKET_SIZE];
+                    match self.fd.read(&mut rem_data[0..size]) {
+                        Ok(b) => {
+                            if b != size {
+                                println!(
+                                    "Size({}) doesn't match bytes read({}). Aborting...",
+                                    size, b
+                                );
+                                return None;
+                            }
+
+                            p.data = rem_data[0..b].to_vec();
+                            Some(p)
+                        }
+                        Err(e) => {
+                            println!("Couldn't read any packet data: {}", e);
+                            None
+                        }
+                    }
+                } else {
+                    Some(p)
+                }
+            }
+            Err(e) => {
+                println!("Failed to parse data: {:?}", e);
+                None
+            }
+        }
+    }
+}
+
+/// What kind of log file is this?
+#[derive(Clone, Debug)]
+pub enum LogType {
+    /// Linux snoop file generated by something like `btmon`.
+    LinuxSnoop(LinuxSnoopHeader),
+}
+
+/// Parses different Bluetooth log types.
+pub struct LogParser {
+    fd: File,
+    log_type: Option<LogType>,
+}
+
+impl<'a> LogParser {
+    pub fn new(filepath: &str) -> std::io::Result<Self> {
+        Ok(Self { fd: File::open(filepath)?, log_type: None })
+    }
+
+    /// Check the log file type for the current log file. This rewinds the position of the file.
+    /// For a non-intrusive query, use |get_log_type|.
+    pub fn read_log_type(&mut self) -> std::io::Result<LogType> {
+        let mut buf = [0; LINUX_SNOOP_HEADER_SIZE];
+
+        // First rewind to start of the file.
+        self.fd.rewind()?;
+        let bytes = self.fd.read(&mut buf)?;
+
+        if let Ok(header) = LinuxSnoopHeader::try_from(&buf[0..bytes]) {
+            let log_type = LogType::LinuxSnoop(header);
+            self.log_type = Some(log_type.clone());
+            Ok(log_type)
+        } else {
+            Err(Error::new(ErrorKind::Other, "Unsupported log file type"))
+        }
+    }
+
+    /// Get cached log type. To initially read the log type, use |read_log_type|.
+    pub fn get_log_type(&self) -> Option<LogType> {
+        self.log_type.clone()
+    }
+
+    pub fn get_snoop_iterator(&mut self) -> Option<LinuxSnoopReader> {
+        // Limit to LinuxSnoop files.
+        if !matches!(self.get_log_type()?, LogType::LinuxSnoop(_)) {
+            return None;
+        }
+
+        Some(LinuxSnoopReader::new(&mut self.fd))
+    }
+}