packages/rag/langchain/LangChain_Local-LLMs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2634dfa3-1263-42b9-901d-004f5166e38e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ggml_init_cublas: found 1 CUDA devices:\n",
      "  Device 0: Orin, compute capability 8.7\n",
      "llama.cpp: loading model from /data/models/text-generation-webui/TheBloke_Llama-2-13B-chat-GGML/llama-2-13b-chat.ggmlv3.q4_0.bin\n",
      "llama_model_load_internal: format     = ggjt v3 (latest)\n",
      "llama_model_load_internal: n_vocab    = 32000\n",
      "llama_model_load_internal: n_ctx      = 2048\n",
      "llama_model_load_internal: n_embd     = 5120\n",
      "llama_model_load_internal: n_mult     = 256\n",
      "llama_model_load_internal: n_head     = 40\n",
      "llama_model_load_internal: n_head_kv  = 40\n",
      "llama_model_load_internal: n_layer    = 40\n",
      "llama_model_load_internal: n_rot      = 128\n",
      "llama_model_load_internal: n_gqa      = 1\n",
      "llama_model_load_internal: rnorm_eps  = 5.0e-06\n",
      "llama_model_load_internal: n_ff       = 13824\n",
      "llama_model_load_internal: freq_base  = 10000.0\n",
      "llama_model_load_internal: freq_scale = 1\n",
      "llama_model_load_internal: ftype      = 2 (mostly Q4_0)\n",
      "llama_model_load_internal: model size = 13B\n",
      "llama_model_load_internal: ggml ctx size =    0.11 MB\n",
      "llama_model_load_internal: using CUDA for GPU acceleration\n",
      "llama_model_load_internal: mem required  =  582.00 MB (+ 1600.00 MB per state)\n",
      "llama_model_load_internal: allocating batch_size x (640 kB + n_ctx x 160 B) = 480 MB VRAM for the scratch buffer\n",
      "llama_model_load_internal: offloading 40 repeating layers to GPU\n",
      "llama_model_load_internal: offloading non-repeating layers to GPU\n",
      "llama_model_load_internal: offloading v cache to GPU\n",
      "llama_model_load_internal: offloading k cache to GPU\n",
      "llama_model_load_internal: offloaded 43/43 layers to GPU\n",
      "llama_model_load_internal: total VRAM used: 8976 MB\n",
      "llama_new_context_with_model: kv self size  = 1600.00 MB\n",
      "AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | \n"
     ]
    }
   ],
   "source": [
    "from langchain.llms import LlamaCpp\n",
    "from langchain.callbacks.manager import CallbackManager\n",
    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n",
    "\n",
    "n_gpu_layers = 128  # Metal set to 1 is enough.\n",
    "n_batch = 512  # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.\n",
    "callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])\n",
    "\n",
    "# Make sure the model path is correct for your system!\n",
    "llm = LlamaCpp(\n",
    "    model_path=\"/data/models/text-generation-webui/TheBloke_Llama-2-13B-chat-GGML/llama-2-13b-chat.ggmlv3.q4_0.bin\",\n",
    "    n_gpu_layers=n_gpu_layers,\n",
    "    n_batch=n_batch,\n",
    "    n_ctx=2048,\n",
    "    f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls\n",
    "    callback_manager=callback_manager,\n",
    "    verbose=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d535c96-c9ff-42cd-b388-5b1d42c1102d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Create an itinerary for a 3-day trip to San Francisco, California. Include the following:\n",
      "Day 1:\n",
      "- Morning: Visit Fisherman's Wharf and take a ferry to Alcatraz Island.\n",
      "- Afternoon: Explore Golden Gate Park and visit the de Young Museum.\n",
      "- Evening: Enjoy dinner at North Beach.\n",
      "\n",
      "Day 2:\n",
      "- Morning: Take a trip to Muir Woods National Monument to see the redwood forests.\n",
      "- Afternoon: Visit the Exploratorium science museum and take a walk along the pier.\n",
      "- Evening: Attend a performance at the San Francisco Symphony.\n",
      "\n",
      "Day 3:\n",
      "- Morning: Take a cable car ride to the top of Nob Hill.\n",
      "- Afternoon: Visit the California Palace of the Legion of Honor and the Japanese Tea Garden.\n",
      "- Evening: Enjoy dinner at one of the many excellent seafood restaurants in the city.\n",
      "\n",
      "Note: This is just a sample itinerary and can be adjusted to fit your interests and schedule. Additionally, consider purchasing a San Francisco CityPASS"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "llama_print_timings:        load time =  1680.75 ms\n",
      "llama_print_timings:      sample time =   201.61 ms /   256 runs   (    0.79 ms per token,  1269.78 tokens per second)\n",
      "llama_print_timings: prompt eval time =  1680.49 ms /    17 tokens (   98.85 ms per token,    10.12 tokens per second)\n",
      "llama_print_timings:        eval time = 16994.37 ms /   255 runs   (   66.64 ms per token,    15.00 tokens per second)\n",
      "llama_print_timings:       total time = 19942.30 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nCreate an itinerary for a 3-day trip to San Francisco, California. Include the following:\\nDay 1:\\n- Morning: Visit Fisherman's Wharf and take a ferry to Alcatraz Island.\\n- Afternoon: Explore Golden Gate Park and visit the de Young Museum.\\n- Evening: Enjoy dinner at North Beach.\\n\\nDay 2:\\n- Morning: Take a trip to Muir Woods National Monument to see the redwood forests.\\n- Afternoon: Visit the Exploratorium science museum and take a walk along the pier.\\n- Evening: Attend a performance at the San Francisco Symphony.\\n\\nDay 3:\\n- Morning: Take a cable car ride to the top of Nob Hill.\\n- Afternoon: Visit the California Palace of the Legion of Honor and the Japanese Tea Garden.\\n- Evening: Enjoy dinner at one of the many excellent seafood restaurants in the city.\\n\\nNote: This is just a sample itinerary and can be adjusted to fit your interests and schedule. Additionally, consider purchasing a San Francisco CityPASS\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm(\"Create an itinerary 3-day trip to San Francisco, California.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d85c2a9c-1771-4533-84a3-afca8489a0e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}