DISCLAIMER: I intend for this guide to be helpful and correct, but I make no guarantees. If you find something is inaccurate or missing, please let me know!
GGML is a C/C++ tensor library for machine learning. It implements:
| # chat_template_txt.py | |
| # Python 3.12.3 | |
| import sys | |
| import argparse | |
| import datetime | |
| from typing import TextIO, Optional, Literal | |
| from transformers import AutoTokenizer |
| #include "arg.h" | |
| #include "common.h" | |
| #include "log.h" | |
| #include "llama.h" | |
| #include "ggml.h" | |
| #include <map> | |
| #include <cstdio> | |
| #include <string> | |
| #include <vector> |
| /// adaptive-p: select tokens near a configurable target probability over time. | |
| /// | |
| /// the adaptive-p sampler transforms the token probability distribution to favor tokens | |
| /// that fall near a user-configurable probability target. | |
| /// | |
| /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* | |
| /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an | |
| /// adapted target probability at each sampling step, thus maintaining the desired target | |
| /// probability over time. | |
| /// |
| # train_tokenizer2.py | |
| # Python 3.12.3 | |
| """Train a `PreTrainedTokenizerFast` given parquet files and a vocabulary size.""" | |
| import sys | |
| import argparse | |
| import tempfile | |
| import shutil | |
| import os |
| import os | |
| import sys | |
| import ctypes | |
| import numpy as np | |
| from easy_llama import libllama as lib | |
| # -------------------------------------------------------------------------------------------- # |
Tip
Skip to the bottom of this document for a TL;DR
For more info, see llama.cpp #12511: Handle user-defined quantization levels for additional tensors by @EAddario
Testing done by @ddh0 using this branch as of committ 5a304b8. Using libllama built for Linux CUDA.
| import os | |
| import torch | |
| import psutil | |
| import datasets | |
| import glob | |
| from transformers import ( | |
| AutoTokenizer, LlamaConfig, LlamaForCausalLM, Trainer, TrainingArguments, | |
| DataCollatorForLanguageModeling | |
| ) |
| # hfget.py | |
| # Python 3.11.2 | |
| # Hugging Face Filesystem API: | |
| # https://huggingface.co/docs/huggingface_hub/en/guides/hf_file_system | |
| import os | |
| import sys | |
| import time | |
| from typing import Union |
| import easy_llama as ez | |
| from subprocess import run | |
| USER = ez.utils.USER_STYLE | |
| BOT = ez.utils.BOT_STYLE | |
| DIM = ez.utils.DIM_STYLE | |
| RESET = ez.utils.RESET_ALL | |
| def say(something: str) -> None: | |
| run(['say', something]) | |
| Llama3 = ez.Model( | |
| 'Meta-Llama-3-8B-Instruct-q8_0.gguf', |
