triton_inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /home/nikhilesh/default_home/miniforge3/lib/python3.12/site-packages/gevent/libuv/_corecffi.abi3.so)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 7\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtritonclient\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhttp\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mhttpclient\u001b[39;00m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# triton_client = httpclient.InferenceServerClient(url=\"10.4.25.9:8011\")\u001b[39;00m\n\u001b[0;32m----> 7\u001b[0m triton_client \u001b[38;5;241m=\u001b[39m \u001b[43mhttpclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mInferenceServerClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlocalhost:8011\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/tritonclient/http/_client.py:182\u001b[0m, in \u001b[0;36mInferenceServerClient.__init__\u001b[0;34m(self, url, verbose, concurrency, connection_timeout, network_timeout, max_greenlets, ssl, ssl_options, ssl_context_factory, insecure)\u001b[0m\n\u001b[1;32m    180\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parsed_url \u001b[38;5;241m=\u001b[39m URL(scheme \u001b[38;5;241m+\u001b[39m url)\n\u001b[1;32m    181\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_base_uri \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_parsed_url\u001b[38;5;241m.\u001b[39mrequest_uri\u001b[38;5;241m.\u001b[39mrstrip(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 182\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_client_stub \u001b[38;5;241m=\u001b[39m \u001b[43mHTTPClient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_url\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    183\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parsed_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    184\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconcurrency\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrency\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    185\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconnection_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconnection_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    186\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnetwork_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnetwork_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    187\u001b[0m \u001b[43m    \u001b[49m\u001b[43mssl_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mssl_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    188\u001b[0m \u001b[43m    \u001b[49m\u001b[43mssl_context_factory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mssl_context_factory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    189\u001b[0m \u001b[43m    \u001b[49m\u001b[43minsecure\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minsecure\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    190\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    191\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool \u001b[38;5;241m=\u001b[39m gevent\u001b[38;5;241m.\u001b[39mpool\u001b[38;5;241m.\u001b[39mPool(max_greenlets)\n\u001b[1;32m    192\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_verbose \u001b[38;5;241m=\u001b[39m verbose\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/geventhttpclient/client.py:68\u001b[0m, in \u001b[0;36mHTTPClient.from_url\u001b[0;34m(cls, url, **kw)\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m enable_ssl:\n\u001b[1;32m     67\u001b[0m     kw\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mssl_options\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m---> 68\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mport\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mssl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menable_ssl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/geventhttpclient/client.py:125\u001b[0m, in \u001b[0;36mHTTPClient.__init__\u001b[0;34m(self, host, port, headers, block_size, connection_timeout, network_timeout, disable_ipv6, concurrency, ssl, ssl_options, ssl_context_factory, insecure, proxy_host, proxy_port, version, headers_type)\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m connection_port:\n\u001b[1;32m    124\u001b[0m         connection_port \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mport\n\u001b[0;32m--> 125\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_connection_pool \u001b[38;5;241m=\u001b[39m \u001b[43mConnectionPool\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    126\u001b[0m \u001b[43m        \u001b[49m\u001b[43mconnection_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconnection_port\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    127\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    128\u001b[0m \u001b[43m        \u001b[49m\u001b[43msize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcurrency\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    129\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnetwork_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnetwork_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    130\u001b[0m \u001b[43m        \u001b[49m\u001b[43mconnection_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconnection_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    131\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdisable_ipv6\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdisable_ipv6\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    132\u001b[0m \u001b[43m        \u001b[49m\u001b[43muse_proxy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43muse_proxy\u001b[49m\n\u001b[1;32m    133\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    134\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mversion \u001b[38;5;241m=\u001b[39m version\n\u001b[1;32m    135\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders_type \u001b[38;5;241m=\u001b[39m headers_type\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/geventhttpclient/connectionpool.py:60\u001b[0m, in \u001b[0;36mConnectionPool.__init__\u001b[0;34m(self, connection_host, connection_port, request_host, request_port, size, disable_ipv6, connection_timeout, network_timeout, use_proxy)\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_request_port \u001b[38;5;241m=\u001b[39m request_port\n\u001b[1;32m     59\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_semaphore \u001b[38;5;241m=\u001b[39m lock\u001b[38;5;241m.\u001b[39mBoundedSemaphore(size)\n\u001b[0;32m---> 60\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_socket_queue \u001b[38;5;241m=\u001b[39m \u001b[43mgevent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mqueue\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLifoQueue\u001b[49m\u001b[43m(\u001b[49m\u001b[43msize\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     61\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_use_proxy \u001b[38;5;241m=\u001b[39m use_proxy\n\u001b[1;32m     63\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconnection_timeout \u001b[38;5;241m=\u001b[39m connection_timeout\n",
      "File \u001b[0;32msrc/gevent/queue.py:154\u001b[0m, in \u001b[0;36mgevent._gevent_cqueue.Queue.__init__\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32msrc/gevent/_hub_local.py:115\u001b[0m, in \u001b[0;36mgevent._gevent_c_hub_local.get_hub_noargs\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/hub.py:445\u001b[0m, in \u001b[0;36mHub.__init__\u001b[0;34m(self, loop, default)\u001b[0m\n\u001b[1;32m    443\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m loop \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    444\u001b[0m         loop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbackend\n\u001b[0;32m--> 445\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mloop \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop_class\u001b[49m(flags\u001b[38;5;241m=\u001b[39mloop, default\u001b[38;5;241m=\u001b[39mdefault) \u001b[38;5;66;03m# pylint:disable=not-callable\u001b[39;00m\n\u001b[1;32m    446\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_resolver \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    447\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_threadpool \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/hub.py:459\u001b[0m, in \u001b[0;36mHub.loop_class\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    457\u001b[0m \u001b[38;5;129m@property\u001b[39m\n\u001b[1;32m    458\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mloop_class\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 459\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mGEVENT_CONFIG\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloop\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/_config.py:55\u001b[0m, in \u001b[0;36mSettingType.__new__.<locals>.getter\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetter\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 55\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msettings\u001b[49m\u001b[43m[\u001b[49m\u001b[43msetting_name\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/_config.py:151\u001b[0m, in \u001b[0;36mSetting.get\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalue\n\u001b[1;32m    149\u001b[0m \u001b[38;5;66;03m# Otherwise, read from the environment and reify\u001b[39;00m\n\u001b[1;32m    150\u001b[0m \u001b[38;5;66;03m# so we return consistent results.\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalue \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalue\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/_config.py:259\u001b[0m, in \u001b[0;36mImportableSetting.validate\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m    257\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(value, \u001b[38;5;28mtype\u001b[39m):\n\u001b[1;32m    258\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m value\n\u001b[0;32m--> 259\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_import_one_of\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshortname_map\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/_config.py:234\u001b[0m, in \u001b[0;36mImportableSetting._import_one_of\u001b[0;34m(self, candidates)\u001b[0m\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[1;32m    232\u001b[0m         \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 234\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_import_one\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcandidates\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/_config.py:248\u001b[0m, in \u001b[0;36mImportableSetting._import_one\u001b[0;34m(self, path, _MISSING)\u001b[0m\n\u001b[1;32m    241\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot import \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    242\u001b[0m                       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRequired format: [package.]module.class. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    243\u001b[0m                       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOr choose from \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    244\u001b[0m                       \u001b[38;5;241m%\u001b[39m (path, \u001b[38;5;28mlist\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshortname_map)))\n\u001b[1;32m    247\u001b[0m module, item \u001b[38;5;241m=\u001b[39m path\u001b[38;5;241m.\u001b[39mrsplit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m--> 248\u001b[0m module \u001b[38;5;241m=\u001b[39m \u001b[43mimportlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mimport_module\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodule\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    249\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(module, item, _MISSING)\n\u001b[1;32m    250\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m x \u001b[38;5;129;01mis\u001b[39;00m _MISSING:\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/importlib/__init__.py:90\u001b[0m, in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m     89\u001b[0m         level \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m---> 90\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_bootstrap\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_gcd_import\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m[\u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpackage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlevel\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap>:1387\u001b[0m, in \u001b[0;36m_gcd_import\u001b[0;34m(name, package, level)\u001b[0m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap>:1360\u001b[0m, in \u001b[0;36m_find_and_load\u001b[0;34m(name, import_)\u001b[0m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap>:1331\u001b[0m, in \u001b[0;36m_find_and_load_unlocked\u001b[0;34m(name, import_)\u001b[0m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap>:935\u001b[0m, in \u001b[0;36m_load_unlocked\u001b[0;34m(spec)\u001b[0m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap_external>:994\u001b[0m, in \u001b[0;36mexec_module\u001b[0;34m(self, module)\u001b[0m\n",
      "File \u001b[0;32m<frozen importlib._bootstrap>:488\u001b[0m, in \u001b[0;36m_call_with_frames_removed\u001b[0;34m(f, *args, **kwds)\u001b[0m\n",
      "File \u001b[0;32m~/default_home/miniforge3/lib/python3.12/site-packages/gevent/libuv/loop.py:23\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgevent\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_ffi\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mloop\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AbstractCallbacks\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgevent\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_interfaces\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ILoop\n\u001b[0;32m---> 23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgevent\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlibuv\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _corecffi \u001b[38;5;66;03m# pylint:disable=no-name-in-module,import-error\u001b[39;00m\n\u001b[1;32m     25\u001b[0m ffi \u001b[38;5;241m=\u001b[39m _corecffi\u001b[38;5;241m.\u001b[39mffi\n\u001b[1;32m     26\u001b[0m libuv \u001b[38;5;241m=\u001b[39m _corecffi\u001b[38;5;241m.\u001b[39mlib\n",
      "\u001b[0;31mImportError\u001b[0m: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /home/nikhilesh/default_home/miniforge3/lib/python3.12/site-packages/gevent/libuv/_corecffi.abi3.so)"
     ]
    }
   ],
   "source": [
    "# pip install tritonclient[http] soundfile\n",
    "import soundfile\n",
    "import numpy as np\n",
    "from tritonclient.utils import *\n",
    "import tritonclient.http as httpclient\n",
    "# triton_client = httpclient.InferenceServerClient(url=\"10.4.25.9:8011\")\n",
    "triton_client = httpclient.InferenceServerClient(url=\"localhost:8011\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " चह भारकेमहर वा ीकावायजकसीर देके नही हलजा ररभारतीय ने यह काम किया हैतो आज तक और तिहाल शिरोमणि का मानन हैकिइस समय मे देश के लि ही सबसे बडी चिंता की बात हैकि भात के लोगक मुसलमान को वोट दे रहे हैाद केलो कहैरानकर ह हरे देश मे कु की कक हो क है क ा है। \n"
     ]
    }
   ],
   "source": [
    "audio, rate = soundfile.read(\"recording.wav\", dtype=\"float32\")\n",
    "audio = np.array(audio, dtype='float32')[1300000:2000000]\n",
    "language = np.array(['hi'], dtype='object')\n",
    "inputs = [\n",
    "    httpclient.InferInput(\n",
    "        \"INPUT_AUDIO\", audio.shape, np_to_triton_dtype(audio.dtype)\n",
    "    ),\n",
    "    httpclient.InferInput(\n",
    "        \"INPUT_LANGUAGE_ID\", language.shape, np_to_triton_dtype(language.dtype)\n",
    "    ),\n",
    "]\n",
    "inputs[0].set_data_from_numpy(audio)\n",
    "inputs[1].set_data_from_numpy(language)\n",
    "outputs = [httpclient.InferRequestedOutput(\"OUTPUT_RECOGNIZED_TEXT\")]\n",
    "result = triton_client.infer(model_name=\"asr\", inputs=inputs, outputs=outputs)\n",
    "result = result.as_numpy(\"OUTPUT_RECOGNIZED_TEXT\")\n",
    "print(result[0].decode('utf-8'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dhruva-iitm-tts",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}