{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "gpuClass": "standard", "widgets": { "application/vnd.jupyter.widget-state+json": { "d5e90ca28fa84e449654b3410fa1d9b1": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_429ac883ced94e6a8523c88425a8375a", "IPY_MODEL_76ddb3d32b5847a18c45d8ddcfab1a41", "IPY_MODEL_515786e0edf2437c802f0d15a32ef9fc" ], "layout": "IPY_MODEL_30bfe0086adc4c65b559628053efd235" } }, "429ac883ced94e6a8523c88425a8375a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a4f8604338bb46818873e0268e24e3a7", "placeholder": "​", "style": "IPY_MODEL_d501bf48c1304e8d87b34cb871e29142", "value": "Downloading pytorch_model.bin: 100%" } }, "76ddb3d32b5847a18c45d8ddcfab1a41": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9279389623ca4f2eaf6efd70df01ba8f", "max": 3893083075, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_9e087b2067e548bda20691817c3f5189", "value": 3893083075 } }, "515786e0edf2437c802f0d15a32ef9fc": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0ff9a42d48334182b9b59a304c29b702", "placeholder": "​", "style": "IPY_MODEL_920ce7efef9d4770b566010b556c367f", "value": " 3.89G/3.89G [01:25<00:00, 60.6MB/s]" } }, "30bfe0086adc4c65b559628053efd235": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a4f8604338bb46818873e0268e24e3a7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d501bf48c1304e8d87b34cb871e29142": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9279389623ca4f2eaf6efd70df01ba8f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9e087b2067e548bda20691817c3f5189": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "0ff9a42d48334182b9b59a304c29b702": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "920ce7efef9d4770b566010b556c367f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "source": [ "## 安装依赖" ], "metadata": { "id": "MZyEDzEHyDcN" } }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "eNbQgYuLxp-3", "outputId": "179d7305-a7d0-4da2-a2f0-ef8eeabe5d2a" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Requirement already satisfied: protobuf==3.20.0 in /usr/local/lib/python3.9/dist-packages (3.20.0)\n", "Requirement already satisfied: transformers==4.27.1 in /usr/local/lib/python3.9/dist-packages (4.27.1)\n", "Collecting icetk\n", " Using cached icetk-0.0.7-py3-none-any.whl (16 kB)\n", "Requirement already satisfied: cpm_kernels in /usr/local/lib/python3.9/dist-packages (1.0.11)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (2.0.0+cu118)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (0.13.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (23.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (6.0)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (2022.10.31)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (4.65.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (3.11.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (2.27.1)\n", "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (0.13.3)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.27.1) (1.22.4)\n", "Requirement already satisfied: torchvision in /usr/local/lib/python3.9/dist-packages (from icetk) (0.15.1+cu118)\n", "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.9/dist-packages (from icetk) (0.1.98)\n", " Using cached icetk-0.0.6-py3-none-any.whl (15 kB)\n", " Using cached icetk-0.0.5-py3-none-any.whl (15 kB)\n", " Using cached icetk-0.0.4-py3-none-any.whl (15 kB)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from torch) (1.11.1)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch) (3.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.5.0)\n", "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.9/dist-packages (from torch) (2.0.0)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from torch) (3.1.2)\n", "Requirement already satisfied: cmake in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (3.25.2)\n", "Requirement already satisfied: lit in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch) (16.0.1)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->torch) (2.1.2)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.27.1) (2022.12.7)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.27.1) (3.4)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.27.1) (1.26.15)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers==4.27.1) (2.0.12)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->torch) (1.3.0)\n", "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.9/dist-packages (from torchvision->icetk) (8.4.0)\n", "Installing collected packages: icetk\n", "Successfully installed icetk-0.0.4\n", "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Collecting fastapi\n", " Using cached fastapi-0.95.1-py3-none-any.whl (56 kB)\n", "Requirement already satisfied: pydantic in /usr/local/lib/python3.9/dist-packages (1.10.7)\n", "Collecting uvicorn\n", " Using cached uvicorn-0.21.1-py3-none-any.whl (57 kB)\n", "Collecting sse_starlette\n", " Using cached sse_starlette-1.3.4-py3-none-any.whl (8.9 kB)\n", "Collecting pyngrok\n", " Using cached pyngrok-6.0.0.tar.gz (681 kB)\n", " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.9/dist-packages (1.5.6)\n", "Collecting starlette<0.27.0,>=0.26.1\n", " Using cached starlette-0.26.1-py3-none-any.whl (66 kB)\n", "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic) (4.5.0)\n", "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.9/dist-packages (from uvicorn) (8.1.3)\n", "Collecting h11>=0.8\n", " Using cached h11-0.14.0-py3-none-any.whl (58 kB)\n", "Requirement already satisfied: PyYAML in /usr/local/lib/python3.9/dist-packages (from pyngrok) (6.0)\n", "Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.9/dist-packages (from starlette<0.27.0,>=0.26.1->fastapi) (3.6.2)\n", "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.9/dist-packages (from anyio<5,>=3.4.0->starlette<0.27.0,>=0.26.1->fastapi) (1.3.0)\n", "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.9/dist-packages (from anyio<5,>=3.4.0->starlette<0.27.0,>=0.26.1->fastapi) (3.4)\n", "Building wheels for collected packages: pyngrok\n", " Building wheel for pyngrok (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for pyngrok: filename=pyngrok-6.0.0-py3-none-any.whl size=19879 sha256=4158360e5a2c7bc1519a6b2a9b98e8595d14000bc0d5a60320ce0ea422ad04d4\n", " Stored in directory: /root/.cache/pip/wheels/31/49/9c/44b13823eb256a3b4dff34b972f7a3c7d9910bfef269e59bd7\n", "Successfully built pyngrok\n", "Installing collected packages: pyngrok, h11, uvicorn, starlette, sse_starlette, fastapi\n", "Successfully installed fastapi-0.95.1 h11-0.14.0 pyngrok-6.0.0 sse_starlette-1.3.4 starlette-0.26.1 uvicorn-0.21.1\n" ] } ], "source": [ "import locale\n", "locale.getpreferredencoding = lambda: \"UTF-8\"\n", "\n", "!pip install protobuf==3.20.0 transformers==4.27.1 icetk cpm_kernels torch\n", "!pip install fastapi pydantic uvicorn sse_starlette pyngrok nest-asyncio" ] }, { "cell_type": "markdown", "source": [ "## 环境配置" ], "metadata": { "id": "VaY7_gNOzdoG" } }, { "cell_type": "code", "source": [ "chatglm_models = [\n", " \"THUDM/chatglm-6b\", # 原始模型\n", " \"THUDM/chatglm-6b-int8\", # int8 量化\n", " \"THUDM/chatglm-6b-int4\", # int4 量化\n", "]\n", "\n", "CHATGLM_MODEL = \"THUDM/chatglm-6b-int4\"\n", "\n", "# GPU/CPU\n", "RUNNING_DEVICE = \"GPU\"\n", "\n", "# API_TOKEN\n", "TOKEN = \"token1\"\n" ], "metadata": { "id": "JJOlpnVOyIeG" }, "execution_count": 2, "outputs": [] }, { "cell_type": "markdown", "source": [ "## 启动模型" ], "metadata": { "id": "CFzHRCMYz08N" } }, { "cell_type": "code", "source": [ "from transformers import AutoModel, AutoTokenizer\n", "\n", "def init_chatglm(model_name: str, running_device: str):\n", " tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n", " model = AutoModel.from_pretrained(model_name, trust_remote_code=True)\n", "\n", " if running_device == \"GPU\":\n", " model = model.half().cuda()\n", " else:\n", " model = model.float()\n", " model.eval()\n", " return tokenizer, model\n", "\n", "tokenizer, model = init_chatglm(CHATGLM_MODEL, RUNNING_DEVICE)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 258, "referenced_widgets": [ "d5e90ca28fa84e449654b3410fa1d9b1", "429ac883ced94e6a8523c88425a8375a", "76ddb3d32b5847a18c45d8ddcfab1a41", "515786e0edf2437c802f0d15a32ef9fc", "30bfe0086adc4c65b559628053efd235", "a4f8604338bb46818873e0268e24e3a7", "d501bf48c1304e8d87b34cb871e29142", "9279389623ca4f2eaf6efd70df01ba8f", "9e087b2067e548bda20691817c3f5189", "0ff9a42d48334182b9b59a304c29b702", "920ce7efef9d4770b566010b556c367f" ] }, "id": "VIT1nTv_1a4X", "outputId": "1f332557-080b-4bac-ac10-720208568a1e" }, "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision.\n", "Explicitly passing a `revision` is encouraged when loading a configuration with custom code to ensure no malicious code has been contributed in a newer revision.\n", "Explicitly passing a `revision` is encouraged when loading a model with custom code to ensure no malicious code has been contributed in a newer revision.\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading pytorch_model.bin: 0%| | 0.00/3.89G [00:00