ParlzMAI: pure-C MoE inference + ParlzAIPlatformPAP framework
- pmai unified CLI (generate/chat/interactive/http/inspect/output/config) - GPT+MoE transformer, .pap (f32/fp16/q8) + GGUF loader (order+version adaptive) - llama/Mixtral arch: RoPE+GQA+SwiGLU+MoE (C==torch verified) - C llama BPE tokenizer (validated vs llama-cpp-python) - training framework + 0.1B/0.22B MoE models; quantization fp16/q8 - build artifacts to output/; HTTP API; config.yaml; scripts; openapi
这个提交包含在：
JGZYES
+22
@@ -0,0 +1,22 @@
|
||||
# 构建/产物
|
||||
build/
|
||||
build-asan/
|
||||
dist/
|
||||
output/
|
||||
|
||||
# WSL 发行版镜像（大）
|
||||
wsl/
|
||||
|
||||
# 模型文件（大，可重新训练/下载）
|
||||
models/*.pap
|
||||
models/*.gguf
|
||||
!models/.gitkeep
|
||||
|
||||
# 缓存/二进制
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
*.core
|
||||
*.papcache
|
||||
__pycache__/
|
||||
*.pyc
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
# moe-serve — 纯 C 的 MoE 推理服务
|
||||
# ============================================
|
||||
# Phase 1: 核心基础 (Tensor / Mat / Math / MemPool / Logger)
|
||||
# Phase 2: GGUF 加载 + MoE 推理引擎
|
||||
# Phase 3: HTTP/REST + WebSocket API (libmicrohttpd)
|
||||
# Phase 4: 部署与监控
|
||||
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
project(moe-serve LANGUAGES C)
|
||||
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_C_EXTENSIONS ON) # gnu11: localtime_r 等 POSIX 函数
|
||||
|
||||
if(NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE Release)
|
||||
endif()
|
||||
# 构建产物统一放到 output/ 目录
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/output)
|
||||
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}/output)
|
||||
|
||||
# ---- 可选依赖探测 ----
|
||||
# OpenMP: 用 -fopenmp 标志（编译+链接）交给活跃编译器解析 libgomp，
|
||||
# 避免某些多 gcc 版本环境下解析成不存在的绝对库路径(如 gcc-11 的 libgomp.so)。
|
||||
find_package(OpenMP QUIET)
|
||||
set(MOE_OPENMP "")
|
||||
set(MOE_HAS_OPENMP OFF)
|
||||
if(OpenMP_C_FOUND)
|
||||
set(MOE_OPENMP "-fopenmp")
|
||||
set(MOE_HAS_OPENMP ON)
|
||||
message(STATUS "OpenMP 已启用 (-fopenmp)")
|
||||
else()
|
||||
message(WARNING "未找到 OpenMP，将串行编译（可后续开启并行）")
|
||||
endif()
|
||||
|
||||
function(moe_apply_openmp tgt)
|
||||
if(MOE_HAS_OPENMP)
|
||||
target_compile_options(${tgt} PRIVATE ${MOE_OPENMP})
|
||||
target_link_options(${tgt} PRIVATE ${MOE_OPENMP})
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# ---- HTTP 服务依赖 (libmicrohttpd / jansson)，默认可选 ----
|
||||
option(BUILD_HTTP "Build HTTP server support (--serve)" OFF)
|
||||
set(MOE_HAVE_HTTP OFF)
|
||||
if(BUILD_HTTP)
|
||||
find_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_check_modules(MO_MHD IMPORTED_TARGET libmicrohttpd)
|
||||
pkg_check_modules(MO_JANSSON IMPORTED_TARGET jansson)
|
||||
if(MO_MHD_FOUND AND MO_JANSSON_FOUND)
|
||||
set(MOE_HAVE_HTTP ON)
|
||||
message(STATUS "HTTP --serve 可用 (libmicrohttpd ${MO_MHD_VERSION} + jansson)")
|
||||
else()
|
||||
message(WARNING "缺少 libmicrohttpd/jansson，--serve 不可用（apt-get install libmicrohttpd-dev libjansson-dev）")
|
||||
endif()
|
||||
else()
|
||||
message(WARNING "未找到 pkg-config，--serve 不可用")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ---- 基础编译选项 ----
|
||||
set(MOE_WARNINGS -Wall -Wextra -Wpedantic)
|
||||
|
||||
# ---- 库：core (tensor + matrix + transformer) ----
|
||||
add_library(moe_core STATIC
|
||||
src/core/tensor.c
|
||||
src/core/matrix.c
|
||||
src/core/transformer.c)
|
||||
target_include_directories(moe_core PUBLIC ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(moe_core PRIVATE ${MOE_WARNINGS} -O3)
|
||||
|
||||
# ---- 库：utils (math + mempool + logger) ----
|
||||
add_library(moe_utils STATIC
|
||||
src/utils/math.c
|
||||
src/utils/mempool.c
|
||||
src/utils/logger.c)
|
||||
target_include_directories(moe_utils PUBLIC ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(moe_utils PRIVATE ${MOE_WARNINGS} -O3)
|
||||
|
||||
# ---- 库：model (config + tokenizer + model/loader + gguf + llama) ----
|
||||
add_library(moe_model STATIC
|
||||
src/model/config.c
|
||||
src/model/tokenizer.c
|
||||
src/model/model.c
|
||||
src/model/gguf.c
|
||||
src/model/llama.c
|
||||
src/model/llama_tokenizer.c)
|
||||
target_include_directories(moe_model PUBLIC ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(moe_model PRIVATE ${MOE_WARNINGS} -O3)
|
||||
|
||||
# ---- 库：infer (sample + generate) ----
|
||||
add_library(moe_infer STATIC
|
||||
src/infer/sample.c
|
||||
src/infer/generate.c)
|
||||
target_include_directories(moe_infer PUBLIC ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(moe_infer PRIVATE ${MOE_WARNINGS} -O3)
|
||||
|
||||
set(MOE_INTERFACE_LIBS moe_infer moe_model moe_core moe_utils m)
|
||||
# 静态库存在循环依赖 (moe_model↔moe_infer)，用链接组解决
|
||||
set(MOE_GROUP "-Wl,--start-group;moe_infer;moe_model;moe_core;moe_utils;-Wl,--end-group;m")
|
||||
|
||||
# ---- 可执行文件 ----
|
||||
add_executable(pmai src/main.c)
|
||||
target_include_directories(pmai PRIVATE ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(pmai PRIVATE ${MOE_WARNINGS} -O3)
|
||||
target_link_libraries(pmai PRIVATE ${MOE_GROUP})
|
||||
moe_apply_openmp(pmai)
|
||||
if(MOE_HAVE_HTTP)
|
||||
target_sources(pmai PRIVATE src/server/api.c)
|
||||
target_link_libraries(pmai PRIVATE PkgConfig::MO_MHD PkgConfig::MO_JANSSON)
|
||||
target_compile_definitions(pmai PRIVATE MOE_HTTP=1)
|
||||
endif()
|
||||
|
||||
# ---- llama 架构运行器（构建到 output/）----
|
||||
add_executable(pmai-llama src/llama_main.c)
|
||||
target_include_directories(pmai-llama PRIVATE ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(pmai-llama PRIVATE ${MOE_WARNINGS} -O3)
|
||||
target_link_libraries(pmai-llama PRIVATE ${MOE_GROUP})
|
||||
moe_apply_openmp(pmai-llama)
|
||||
|
||||
# ---- 测试 ----
|
||||
enable_testing()
|
||||
add_executable(test_core tests/test_core.c)
|
||||
target_include_directories(test_core PRIVATE ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(test_core PRIVATE ${MOE_WARNINGS} -O2)
|
||||
target_link_libraries(test_core PRIVATE ${MOE_GROUP})
|
||||
moe_apply_openmp(test_core)
|
||||
add_test(NAME core COMMAND test_core)
|
||||
|
||||
# ---- llama 分词器独立测试 ----
|
||||
add_executable(test_llmtok tests/test_llmtok.c)
|
||||
target_include_directories(test_llmtok PRIVATE ${CMAKE_SOURCE_DIR}/src)
|
||||
target_compile_options(test_llmtok PRIVATE ${MOE_WARNINGS} -O2)
|
||||
target_link_libraries(test_llmtok PRIVATE ${MOE_GROUP})
|
||||
moe_apply_openmp(test_llmtok)
|
||||
@@ -0,0 +1,148 @@
|
||||
# ParlzMAI · ParlzAIPlatformPAP
|
||||
|
||||
> **ParlzMAI**（`pmai`）：纯 C 的 **MoE 推理服务** + 自研 AI 框架 **ParlzAIPlatformPAP**。
|
||||
> 自研模型格式 **`.pap`**、兼容 **GGUF(llama.cpp)**，在 WSL(自带 GPU) 上**真实训练出 0.1B MoE 模型**并端到端生成。
|
||||
|
||||
## 项目性质
|
||||
- 纯 C 推理引擎（`src/`）+ Python 训练框架（`platform/`）+ 统一 CLI 入口 **`pmai`**。
|
||||
- 支持两种模型：自研 GPT+MoE 架构（`.pap`）与 **llama/Mixtral 架构**（GGUF）。
|
||||
- 自定义 `.pap` 格式（`PAP1` 魔数 + 配置 + byte-level BPE 词表 + 命名张量，支持 f32/fp16/q8 量化）。
|
||||
- 训练端可训 0.1B 级全层 MoE 模型，导出 `.pap` 或标准 `.gguf`。
|
||||
|
||||
## 架构
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ pmai (统一 CLI) │
|
||||
│ 生成 · --interactive 对话 · --serve HTTP/SSE │
|
||||
│ --inspect GGUF · --output 落盘 │
|
||||
├──────────────────────────────────────────────┤
|
||||
│ 推理引擎 (src/) C │
|
||||
│ GPT: RMSNorm · Attention+KV · MoE(top-k) │
|
||||
│ llama: RoPE · GQA · SwiGLU · Mixtral MoE │
|
||||
│ 加载器: .pap(多dtype)/GGUF(多量化) │
|
||||
│ 分词器: byte-level BPE / GPT-2 BPE │
|
||||
│ 采样: top-k/temperature │
|
||||
├──────────────────────────────────────────────┤
|
||||
│ 训练框架 (platform/) Python │
|
||||
│ GPT+MoE/pt · BPE · .pap/GGUF 读写 · 量化 │
|
||||
├──────────────────────────────────────────────┤
|
||||
│ WSL2(Ubuntu 22.04) + GPU RTX5060/CUDA │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## 目录结构
|
||||
```
|
||||
src/ ParlzMAI C 引擎
|
||||
core/ tensor/matrix + transformer(GPT) + KVCache
|
||||
model/ config + tokenizer + model(.pap) + gguf(容器/反量化) + llama(架构)
|
||||
infer/ sample(top-k) + generate
|
||||
server/ HTTP 服务 (libmicrohttpd + SSE)
|
||||
utils/ math/mempool/logger
|
||||
platform/ ParlzAIPlatformPAP (Python)
|
||||
tokenizer.py pap_format.py gguf_format.py model.py llama_model.py llama_tokenizer.py
|
||||
train.py export_gguf.py quant_pap.py verify.py verify_llama.py make_random.py
|
||||
data/ 语料 (tinyshakespeare.txt) models/ .pap/.gguf 模型
|
||||
output/ 构建产物 (pmai / pmai-llama / libmoe_*) scripts/ 脚本
|
||||
```
|
||||
|
||||
## 构建与运行（WSL，Ubuntu-22.04）
|
||||
```bash
|
||||
cd /mnt/f/ParlzMAI
|
||||
cmake -S . -B build && cmake --build build -j
|
||||
ctest --test-dir build # 单元测试
|
||||
|
||||
# 构建产物统一在 output/ 目录：pmai(统一入口) / pmai-llama(llama 运行器)
|
||||
./output/pmai --model models/moe-0.1b-fp16.pap --prompt "ROMEO:" --n_tokens 120 --temperature 0.8 --top_k 40
|
||||
./output/pmai --model models/moe-0.1b-fp16.pap --interactive # 对话式
|
||||
./output/pmai --model models/moe-0.1b-fp16.pap --serve --port 8080 # HTTP/SSE
|
||||
./output/pmai --model models/moe-0.1b-fp16.pap --prompt "QUEEN:" --output output/out.txt
|
||||
./output/pmai --inspect models/moe-0.1b.gguf # 查看 GGUF
|
||||
```
|
||||
|
||||
## 训练（GPU）
|
||||
```bash
|
||||
/opt/pap-venv/bin/python platform/train.py --data data/tinyshakespeare.txt \
|
||||
--out models/moe-0.1b.pap --vocab 2048 --layers 6 --d_model 512 --heads 8 \
|
||||
--experts 16 --top_k 2 --d_expert 1024 --max_seq 256 --steps 2500 --batch 8
|
||||
# 约 109M 参数(0.1B)，全层 MoE
|
||||
```
|
||||
|
||||
## 量化 / 格式转换
|
||||
| 操作 | 命令 |
|
||||
|---|---|
|
||||
| `.pap` → fp16 (≈2×) | `python platform/quant_pap.py --src a.pap --dst a-fp16.pap --dtype fp16` |
|
||||
| `.pap` → q8 (≈3.8×) | `python platform/quant_pap.py --src a.pap --dst a-q8.pap --dtype q8` |
|
||||
| `.pap` → `.gguf` | `python platform/export_gguf.py --src a.pap --dst a.gguf --dtype fp16` |
|
||||
|
||||
实测（自研 0.1B = 109,238,784 参数）：
|
||||
|
||||
| 文件 | 体积 | 相对 f32 | 贪心输出 |
|
||||
|---|---|---|---|
|
||||
| `moe-0.1b.pap` (f32) | 437.0 MB | 1.0× | 基准 |
|
||||
| `moe-0.1b-fp16.pap` | 218.5 MB | 2.0× | 一致 |
|
||||
| `moe-0.1b-q8.pap` | 116.1 MB | 3.76× | 一致 |
|
||||
| `moe-0.1b.gguf`(fp16)| 218.5 MB | 2.0× | 一致 |
|
||||
|
||||
## 一致性验证
|
||||
```bash
|
||||
# GPT-MoE：C vs torch 贪心
|
||||
/opt/pap-venv/bin/python platform/verify.py --pap models/_smoke.pap # [MATCH] YES
|
||||
# llama 架构：C vs torch 贪心（合成 MoE GGUF round-trip）
|
||||
/opt/pap-venv/bin/python platform/verify_llama.py # [MATCH] YES
|
||||
# llama BPE 分词器 vs llama-cpp-python（真实 Qwen2-MoE 词表）
|
||||
/opt/pap-venv/bin/python -c "..." # 逐 token MATCH
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# 项目状态
|
||||
|
||||
## ✅ 已实现（已验证）
|
||||
|
||||
### 环境与基建
|
||||
- WSL2 专用镜像 **Ubuntu 22.04** + 工具链（gcc/cmake/make/pip）。
|
||||
- **GPU 直通**：RTX 5060 / 8GB，CUDA 可用；venv `/opt/pap-venv`（torch 2.14+cu130 / numpy）。
|
||||
- **llama-cpp-python**（参考/逐 token 对照）+ **HF 镜像**（可下载真实模型）。
|
||||
|
||||
### 推理引擎（C，`src/`）
|
||||
- ✅ tensor / matrix / math / mempool / logger；`-Wall -Wextra` 干净编译；ctest 全绿。
|
||||
- ✅ **GPT-MoE 前向**：RMSNorm、带 KV Cache 注意力、MoE(router+Top-K+专家加权)、GFU FFN。
|
||||
- ✅ **llama/Mixtral 架构**：RoPE(NeoX) + GQA + SwiGLU + Mixtral MoE(重归一化 Top-k) —— **C 与 PyTorch 贪心逐 token 一致（`[MATCH] YES`）**。
|
||||
- ✅ **byte-level BPE**（GPT/自研两端一致）与 **GPT-2 byte-level BPE 分词器**（`platform/llama_tokenizer.py`，**与 llama-cpp-python 逐 token MATCH，含中文**）。
|
||||
- ✅ **`.pap` 加载器**（多 dtype f32/fp16/q8 反量化）；**GGUF 加载器**（容器解析、元数据查询、F32/F16/BF16/Q4_0/Q5_0/Q5_1/Q8_0 反量化）。
|
||||
- ✅ **真实 GGUF 兼容**：**顺序探测**（元数据/张量在前）+ **版本自适应**（v1 用 u32、v2+ 用 u64）。
|
||||
- ✅ 采样（top-k/temperature）+ 自回归生成。
|
||||
|
||||
### CLI（`pmai`，构建到 `output/`）
|
||||
- ✅ `--prompt` 生成、**`--interactive` 对话**（输入一行→流式生成→exit 退出）、**`--serve` HTTP/SSE**（`POST /api/generate`、`GET /api/models`、`GET /health`）、**`--inspect` 查看任意 GGUF**、**`--output <file>` 落盘**（默认存 `output/`，交互自动 `output/chat-<ts>.txt`）。
|
||||
- ✅ `pmai-llama`：llama 架构 GGUF 运行器（加载 + 生成 id）。
|
||||
|
||||
### 训练框架（Python，`platform/`）
|
||||
- ✅ BPE 训练/编码（O(n log n)）；GPT+MoE(PyTorch)；`.pap`/`gguf` 读写；量化（fp16 ≈2×、q8 ≈3.76×）；导出 `.pap`/`.gguf`。
|
||||
- ✅ **真实训练并验证**：GPU 训练 **0.1B（109M）MoE**（tiny_shakespeare，2500 步 ≈11 分钟，loss 7.79→3.73），产出 `.pap`(437MB)/fp16(218MB)/q8(116MB)/`.gguf`(218MB)。
|
||||
|
||||
## ⬜ 未实现 / 待办（如实）
|
||||
|
||||
| 类别 | 内容 |
|
||||
|---|---|
|
||||
| **qwen2moe 架构运行** | 真实 `Qwen2.5-MoE-2X1.5B`(Q4_K_M) 需：**K 系反量化**(Q4_K/Q5_K/Q6_K/Q8_K/Q2_K/Q3_K，已取到 llama.cpp 源码未移植) + **qwen2moe 分支**(共享专家 `ffn_*_shexp` / 3D 路由专家 `ffn_*_exps` / 双重 router / 注意力 bias)。**未发布未经验证实现**。 |
|
||||
| **C 端 llama 分词器** | `llama_tokenizer.py` 已验证，但 C 端未接入 → `pmai`/`pmai-llama` 对 llama 系列暂输出 id、非文本。 |
|
||||
| **llama 真实文本生成对拍** | 需上两条都完成后，用 `pmai-llama` 与 llama-cpp-python 逐 token 一致才算跑通。 |
|
||||
| **推理增强** | 无 batch 推理、无 prefill 批量化、无 mmap 大模型加载、无专家预加载/热切换。 |
|
||||
| **Phase 3 剩余** | HTTP/SSE/健康检查已做；缺 WebSocket、线程池/请求调度、并发限流。 |
|
||||
| **Phase 4 运维** | 无 config.yaml、无 QPS/延迟监控、无优雅关闭/热重载、无模型下载工具、无 wrk/ab 基准、无 OpenAPI 文档。 |
|
||||
| **精简** | GGUF v1 旧格式模型（如 `klosax` 的 v1 目录）未能完全解析（不影响现代 v2/v3）。 |
|
||||
| **Tokenizer 特殊 token** | `n_special=0`，未内置 `<|im_start|>` 等。 |
|
||||
| **交互上下文** | `--interactive` 为无状态（每轮仅用当前输入，不携带上文）。 |
|
||||
| **训练质量** | 语料 1MB、2500 步，仅“莎士比亚腔”，要更连贯需更大语料 + 长训练。 |
|
||||
|
||||
## 下一步（按优先级）
|
||||
1. **C 端 llama BPE 分词器** → `pmai`/`pmai-llama` 对任何 llama/GPT-2 系 GGUF 输出真实文本（边界清晰、可逐 token 验证）。
|
||||
2. **K 系反量化移植**（Q4_K/Q5_K/Q6_K/Q8_K 等，从 llama.cpp 源码逐行核对）。
|
||||
3. **qwen2moe 架构分支** + 与 llama-cpp-python 端到端对拍 → `pmai-llama` 真跑 `Qwen2.5-MoE-2X1.5B`。
|
||||
4. 批推理 / mmap / QPS 监控 / config.yaml / 模型下载。
|
||||
|
||||
## 环境备注
|
||||
- 构建/训练跑在 WSL `Ubuntu-22.04`：`wsl -d Ubuntu-22.04`。
|
||||
- 项目 Windows `F:\ParlzMAI` ↔ WSL `/mnt/f/ParlzMAI`。
|
||||
- 预处理缓存 `data/tinyshakespeare.txt.papcache`（BPE，避免重跑 ~4.6 分钟）。
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
# pmai 示例配置 config.yaml（--config 加载，命令行参数优先）
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 11434
|
||||
|
||||
model: models/moe-0.1b-fp16.pap # 也支持 .gguf
|
||||
|
||||
inference:
|
||||
n_tokens: 100
|
||||
temperature: 0.8
|
||||
top_k: 40
|
||||
+21915
文件差异内容过多而无法显示
加载差异
+3115
文件差异内容过多而无法显示
加载差异
+40000
文件差异内容过多而无法显示
加载差异
@@ -0,0 +1,51 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: ParlzMAI API
|
||||
version: 1.0.0
|
||||
description: pmai HTTP 服务（--serve，需 -DBUILD_HTTP=ON）。MoE 推理生成。
|
||||
servers:
|
||||
- url: http://127.0.0.1:11434
|
||||
paths:
|
||||
/health:
|
||||
get:
|
||||
summary: 健康检查
|
||||
responses:
|
||||
'200': { description: ok }
|
||||
/api/models:
|
||||
get:
|
||||
summary: 已加载模型信息
|
||||
responses:
|
||||
'200':
|
||||
description: JSON 模型信息
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
model: { type: string }
|
||||
params: { type: integer }
|
||||
arch: { type: string }
|
||||
n_experts: { type: integer }
|
||||
/api/generate:
|
||||
post:
|
||||
summary: 生成（SSE 流式）
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
prompt: { type: string, example: 'ROMEO:' }
|
||||
options:
|
||||
type: object
|
||||
properties:
|
||||
max_tokens: { type: integer, example: 60 }
|
||||
temperature: { type: number, example: 0.8 }
|
||||
top_k: { type: integer, example: 40 }
|
||||
responses:
|
||||
'200':
|
||||
description: SSE text/event-stream，每事件 data: {"response":"..."}
|
||||
content:
|
||||
text/event-stream:
|
||||
schema: { type: string }
|
||||
@@ -0,0 +1,41 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""把自研 MoE 模型的 .pap 导出成标准 GGUF 容器。
|
||||
|
||||
用法:
|
||||
/opt/pap-venv/bin/python platform/export_gguf.py \
|
||||
--src models/moe-0.1b.pap --dst models/moe-0.1b.gguf --dtype fp16 # 或 q8 / f32
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import pap_format
|
||||
import gguf_format
|
||||
|
||||
GML = {"f32": 0, "fp16": 1, "q8": 8}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--src", required=True)
|
||||
ap.add_argument("--dst", required=True)
|
||||
ap.add_argument("--dtype", default="fp16", choices=GML.keys())
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg, vocab, merges, tensors = pap_format.read_pap(args.src)
|
||||
gguf_format.save_gguf(args.dst, cfg, vocab, merges, tensors, GML[args.dtype])
|
||||
|
||||
in_sz = os.path.getsize(args.src)
|
||||
out_sz = os.path.getsize(args.dst)
|
||||
print(f"已导出: {args.src} ({in_sz/1e6:.1f} MB) -> {args.dst} ({out_sz/1e6:.1f} MB)")
|
||||
|
||||
# 读回验证
|
||||
rt, meta, ver = gguf_format.read_gguf(args.dst)
|
||||
print(f"读回校验: version={ver} tensors={len(rt)} metadata={len(meta)}")
|
||||
print(f" general.architecture = {meta.get('general.architecture')}")
|
||||
print(f" pap.vocab_size = {meta.get('pap.vocab_size')} pap.moe_n_experts = {meta.get('pap.moe_n_experts')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,404 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
GGUF (llama.cpp) v2/v3 容器 读取/写入，用于把 ParlzAIPlatformPAP 的自研 MoE 模型
|
||||
导出成标准 GGUF，并能在 ParlzMAI (C 引擎) 中加载运行。也支持读任意 GGUF 做查看/量化分析。
|
||||
|
||||
张量命名沿用 .pap 的名字。元数据约定：
|
||||
pap.* (见 model.py / C 端 model_load_gguf)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import pap_format as pf
|
||||
except Exception: # pragma: no cover
|
||||
pf = None
|
||||
|
||||
_M = "<"
|
||||
MAGIC = b"GGUF"
|
||||
|
||||
# ggml 类型
|
||||
GML_F32 = 0
|
||||
GML_F16 = 1
|
||||
GML_Q4_0 = 2
|
||||
GML_Q5_0 = 6
|
||||
GML_Q5_1 = 7
|
||||
GML_Q8_0 = 8
|
||||
|
||||
# gguf 元数据值类型
|
||||
GV_U8 = 0
|
||||
GV_U32 = 4
|
||||
GV_F32 = 6
|
||||
GV_STRING = 8
|
||||
GV_ARRAY = 9
|
||||
TYPE_NAME = {0: "F32", 1: "F16", 2: "Q4_0", 6: "Q5_0", 7: "Q5_1", 8: "Q8_0"}
|
||||
|
||||
|
||||
def _u64(v):
|
||||
return struct.pack(_M + "Q", int(v))
|
||||
def _u32(v):
|
||||
return struct.pack(_M + "I", int(v) & 0xFFFFFFFF)
|
||||
def _f32(v):
|
||||
return struct.pack(_M + "f", float(v))
|
||||
def _gstr(s: str) -> bytes:
|
||||
b = s.encode("utf-8")
|
||||
return _u64(len(b)) + b
|
||||
|
||||
|
||||
def _meta(vt: int, value) -> bytes:
|
||||
if vt == GV_U8:
|
||||
return bytes([int(value)])
|
||||
if vt == GV_U32:
|
||||
return _u32(value)
|
||||
if vt == GV_F32:
|
||||
return _f32(value)
|
||||
if vt == GV_STRING:
|
||||
return _gstr(value)
|
||||
if vt == GV_ARRAY:
|
||||
inner, arr = value
|
||||
if inner == GV_U8:
|
||||
bb = bytes(arr)
|
||||
return _u32(GV_U8) + _u64(len(bb)) + bb
|
||||
if inner == GV_U32:
|
||||
return _u32(GV_U32) + _u64(len(arr)) + b"".join(_u32(x) for x in arr)
|
||||
if inner == GV_STRING:
|
||||
return _u32(GV_STRING) + _u64(len(arr)) + b"".join(_gstr(s) for s in arr)
|
||||
raise ValueError("不支持的数组内类型")
|
||||
raise ValueError(f"未知元数据类型 {vt}")
|
||||
|
||||
|
||||
def _tensor_bytes(arr: np.ndarray, gml_type: int) -> bytes:
|
||||
w = np.ascontiguousarray(arr, dtype=np.float32).reshape(-1)
|
||||
n = w.size
|
||||
if gml_type == GML_F32:
|
||||
return w.tobytes()
|
||||
if gml_type == GML_F16:
|
||||
return w.astype(np.float16).astype("<f2").tobytes()
|
||||
if gml_type == GML_Q8_0:
|
||||
pad = (-n) % 32
|
||||
wp = np.pad(w, (0, pad), "constant") if pad else w
|
||||
nb = wp.size // 32
|
||||
b = wp.reshape(nb, 32)
|
||||
maxabs = np.maximum(np.abs(b).max(axis=1), 1e-12)
|
||||
d = (maxabs / 127.0).astype(np.float16)
|
||||
# 每块: [fp16 d][32 int8]
|
||||
out = bytearray()
|
||||
for i in range(nb):
|
||||
di = d[i]
|
||||
q = np.clip(np.round(b[i] / float(di) / 1.0), -127, 127).astype(np.int8)
|
||||
out += np.float16(di).astype("<f2").tobytes()
|
||||
out += q.tobytes()
|
||||
return bytes(out)
|
||||
raise ValueError(f"未知 ggml 类型 {gml_type}")
|
||||
|
||||
|
||||
def save_gguf(path: str, cfg, vocab: List[bytes], merges: List[Tuple[int, int]],
|
||||
tensors: Dict[str, np.ndarray], gml_type: int = GML_F16) -> None:
|
||||
"""导出 .gguf。cfg 为 pap_format.ModelConfig。gml_type 选 F16/Q8_0/F32。"""
|
||||
from pap_format import ModelConfig
|
||||
rows, cols = (cfg.d_model, cfg.vocab_size) # 占位断言，未用
|
||||
names = sorted(tensors.keys())
|
||||
|
||||
# 计算张量表与 data 偏移
|
||||
tensor_meta = []
|
||||
data_chunks = []
|
||||
offset = 0
|
||||
for name in names:
|
||||
w = np.ascontiguousarray(tensors[name], dtype=np.float32)
|
||||
if w.ndim == 1:
|
||||
w = w.reshape(1, -1)
|
||||
r, c = w.shape
|
||||
encoded = _tensor_bytes(w, gml_type)
|
||||
# ggml dims: [ne0=列, ne1=行]
|
||||
tensor_meta.append((name, [c, r], gml_type, offset))
|
||||
data_chunks.append(encoded)
|
||||
offset += len(encoded)
|
||||
|
||||
# 元数据
|
||||
meta_kv: List[Tuple[str, int, object]] = [
|
||||
("general.architecture", GV_STRING, "moe-pap"),
|
||||
("general.name", GV_STRING, "ParlzAIPlatformPAP MoE 0.1B"),
|
||||
("pap.vocab_size", GV_U32, cfg.vocab_size),
|
||||
("pap.n_layer", GV_U32, cfg.n_layer),
|
||||
("pap.d_model", GV_U32, cfg.d_model),
|
||||
("pap.n_head", GV_U32, cfg.n_head),
|
||||
("pap.d_ff", GV_U32, cfg.d_ff),
|
||||
("pap.moe_n_experts", GV_U32, cfg.moe_n_experts),
|
||||
("pap.moe_top_k", GV_U32, cfg.moe_top_k),
|
||||
("pap.d_expert", GV_U32, cfg.d_expert),
|
||||
("pap.max_seq_len", GV_U32, cfg.max_seq_len),
|
||||
("pap.num_merges", GV_U32, cfg.num_merges),
|
||||
("pap.rmsnorm_eps", GV_F32, cfg.rmsnorm_eps),
|
||||
("pap.moe_mask", GV_ARRAY, (GV_U8, cfg.moe_mask)),
|
||||
("pap.vocab_len", GV_ARRAY, (GV_U32, [len(t) for t in vocab])),
|
||||
("pap.vocab_blob", GV_ARRAY, (GV_U8, b"".join(vocab))),
|
||||
("pap.merges_a", GV_ARRAY, (GV_U32, [a for a, _ in merges])),
|
||||
("pap.merges_b", GV_ARRAY, (GV_U32, [b for _, b in merges])),
|
||||
]
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(MAGIC)
|
||||
f.write(_u32(3)) # version
|
||||
f.write(_u64(len(tensor_meta))) # tensor_count
|
||||
f.write(_u64(len(meta_kv))) # metadata_kv_count
|
||||
|
||||
for (k, vt, val) in meta_kv:
|
||||
f.write(_gstr(k))
|
||||
f.write(_u32(vt))
|
||||
f.write(_meta(vt, val))
|
||||
|
||||
for (name, dims, t, off) in tensor_meta:
|
||||
f.write(_gstr(name))
|
||||
f.write(_u32(len(dims)))
|
||||
for d in dims:
|
||||
f.write(_u64(d))
|
||||
f.write(_u32(t))
|
||||
f.write(_u64(off))
|
||||
|
||||
# data 段（顺序与 tensor 表一致）
|
||||
for chunk in data_chunks:
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
def read_gguf(path: str):
|
||||
"""读取 GGUF，返回 (tensors: dict, metadata: dict)。用于验证/查看。"""
|
||||
data = open(path, "rb").read()
|
||||
off = 0
|
||||
|
||||
def u32(o):
|
||||
return struct.unpack_from(_M + "I", data, o)[0], o + 4
|
||||
def u64(o):
|
||||
return struct.unpack_from(_M + "Q", data, o)[0], o + 8
|
||||
def gstr(o):
|
||||
n, o = u64(o)
|
||||
return data[o:o + n].decode("utf-8"), o + n
|
||||
|
||||
assert data[0:4] == MAGIC
|
||||
version, off = u32(4)
|
||||
tensor_count, off = u64(8)
|
||||
meta_count, off = u64(16)
|
||||
|
||||
# 探测顺序：张量名总以 .weight 结尾
|
||||
n0, po = u64(off)
|
||||
tensor_first = data[po:po + n0].endswith(b".weight")
|
||||
|
||||
def _meta_loop(off):
|
||||
metadata = {}
|
||||
for _ in range(meta_count):
|
||||
k, off = gstr(off)
|
||||
vt, off = u32(off)
|
||||
v, off = _read_val(data, off, vt)
|
||||
metadata[k] = v
|
||||
return metadata, off
|
||||
|
||||
def _tensor_loop(off):
|
||||
tm = []
|
||||
for _ in range(tensor_count):
|
||||
name, off = gstr(off)
|
||||
ndims, off = u32(off)
|
||||
dims = []
|
||||
for _ in range(ndims):
|
||||
d, off = u64(off)
|
||||
dims.append(d)
|
||||
t, off = u32(off)
|
||||
o, off = u64(off)
|
||||
tm.append((name, dims, t, o))
|
||||
return tm, off
|
||||
|
||||
if tensor_first:
|
||||
tensor_meta, off = _tensor_loop(off)
|
||||
metadata, off = _meta_loop(off)
|
||||
else:
|
||||
metadata, off = _meta_loop(off)
|
||||
tensor_meta, off = _tensor_loop(off)
|
||||
|
||||
data_off = off
|
||||
tensors = {}
|
||||
for (name, dims, t, o) in tensor_meta:
|
||||
n = 1
|
||||
for d in dims:
|
||||
n *= d
|
||||
arr = _dequant(data, data_off + o, n, t)
|
||||
if len(dims) >= 2:
|
||||
arr = arr.reshape(dims[1], dims[0]) # ggml 行主序: (ne1 rows, ne0 cols) = (in,out)
|
||||
tensors[name] = arr
|
||||
return tensors, metadata, version
|
||||
|
||||
|
||||
def read_gguf_meta(path: str):
|
||||
"""只解析 header + 元数据 + 张量信息，不做反量化（用于读分词器/结构）。"""
|
||||
data = open(path, "rb").read()
|
||||
off = 0
|
||||
|
||||
def u32(o):
|
||||
return struct.unpack_from(_M + "I", data, o)[0], o + 4
|
||||
def u64(o):
|
||||
return struct.unpack_from(_M + "Q", data, o)[0], o + 8
|
||||
def gstr(o):
|
||||
n, o = u64(o)
|
||||
return data[o:o + n].decode("utf-8", errors="replace"), o + n
|
||||
|
||||
assert data[0:4] == MAGIC
|
||||
version, off = u32(4)
|
||||
tensor_count, off = u64(8)
|
||||
meta_count, off = u64(16)
|
||||
n0, po = u64(off)
|
||||
tensor_first = data[po:po + n0].endswith(b".weight")
|
||||
|
||||
def _meta(off):
|
||||
md = {}
|
||||
for _ in range(meta_count):
|
||||
k, off = gstr(off)
|
||||
vt, off = u32(off)
|
||||
v, off = _read_val(data, off, vt)
|
||||
md[k] = v
|
||||
return md, off
|
||||
|
||||
def _tens(off):
|
||||
tm = []
|
||||
for _ in range(tensor_count):
|
||||
name, off = gstr(off)
|
||||
nd, off = u32(off)
|
||||
dims = []
|
||||
for _ in range(nd):
|
||||
d, off = u64(off)
|
||||
dims.append(d)
|
||||
t, off = u32(off)
|
||||
o, off = u64(off)
|
||||
tm.append((name, dims, t, o))
|
||||
return tm, off
|
||||
|
||||
if tensor_first:
|
||||
tmeta, off = _tens(off)
|
||||
md, off = _meta(off)
|
||||
else:
|
||||
md, off = _meta(off)
|
||||
tmeta, off = _tens(off)
|
||||
return md, tmeta, version
|
||||
|
||||
|
||||
def _read_val(data, off, vt):
|
||||
def _u32(o):
|
||||
return struct.unpack_from(_M + "I", data, o)[0], o + 4
|
||||
if vt == GV_U8:
|
||||
return data[off], off + 1
|
||||
if vt == 1: # INT8
|
||||
return struct.unpack_from(_M + "b", data, off)[0], off + 1
|
||||
if vt == 2: # UINT16
|
||||
return struct.unpack_from(_M + "H", data, off)[0], off + 2
|
||||
if vt == 3: # INT16
|
||||
return struct.unpack_from(_M + "h", data, off)[0], off + 2
|
||||
if vt == GV_U32:
|
||||
return struct.unpack_from(_M + "I", data, off)[0], off + 4
|
||||
if vt == 5: # INT32
|
||||
return struct.unpack_from(_M + "i", data, off)[0], off + 4
|
||||
if vt == GV_F32:
|
||||
return struct.unpack_from(_M + "f", data, off)[0], off + 4
|
||||
if vt == 7: # BOOL
|
||||
return bool(data[off]), off + 1
|
||||
if vt == GV_STRING:
|
||||
n, o = struct.unpack_from(_M + "Q", data, off)[0], off + 8
|
||||
return data[o:o + n].decode("utf-8", errors="replace"), o + n
|
||||
if vt == GV_ARRAY:
|
||||
inner, o = _u32(off)
|
||||
n, o = struct.unpack_from(_M + "Q", data, o)[0], o + 8
|
||||
items = []
|
||||
for _ in range(n):
|
||||
it, o = _read_val(data, o, inner)
|
||||
items.append(it)
|
||||
return items, o
|
||||
if vt == 10 or vt == 11: # UINT64 / INT64
|
||||
return struct.unpack_from(_M + "q", data, off)[0], off + 8
|
||||
if vt == 12: # FLOAT64
|
||||
return struct.unpack_from(_M + "d", data, off)[0], off + 8
|
||||
raise ValueError(f"未知元数据类型 {vt}")
|
||||
|
||||
|
||||
def _dequant(data, off, n, gml_type):
|
||||
if gml_type == GML_F32:
|
||||
return np.frombuffer(data, np.float32, count=n, offset=off).astype(np.float32)
|
||||
if gml_type == GML_F16:
|
||||
return np.frombuffer(data, np.float16, count=n, offset=off).astype(np.float32)
|
||||
if gml_type == GML_Q8_0:
|
||||
nb = (n + 31) // 32
|
||||
vals = np.empty(n, dtype=np.float32)
|
||||
po = off
|
||||
k = 0
|
||||
for _ in range(nb):
|
||||
scale = np.frombuffer(data, np.float16, count=1, offset=po).astype(np.float32)[0]
|
||||
po += 2
|
||||
cnt = min(32, n - k)
|
||||
q = np.frombuffer(data, np.int8, count=cnt, offset=po)
|
||||
po += cnt
|
||||
vals[k:k + cnt] = q.astype(np.float32) * scale
|
||||
k += cnt
|
||||
return vals
|
||||
raise ValueError(f"不支持读取 ggml 类型 {gml_type}")
|
||||
|
||||
|
||||
def save_llama_gguf(path: str, cfg, tensors: Dict[str, "object"], gml_type: int = GML_F16,
|
||||
tokens=None, merges=None, bos: int = 1, eos: int = 2,
|
||||
name: str = "llama-moe"):
|
||||
"""写 llama/Mixtral 架构的 GGUF。cfg 为 platform.llama_model.LlamaConfig。"""
|
||||
import numpy as np
|
||||
names = sorted(tensors.keys())
|
||||
tensor_meta = []
|
||||
data_chunks = []
|
||||
offset = 0
|
||||
for nm in names:
|
||||
w = np.ascontiguousarray(tensors[nm], dtype=np.float32)
|
||||
if w.ndim == 1:
|
||||
w = w.reshape(1, -1)
|
||||
r, c = w.shape
|
||||
enc = _tensor_bytes(w, gml_type)
|
||||
tensor_meta.append((nm, [c, r], gml_type, offset))
|
||||
data_chunks.append(enc)
|
||||
offset += len(enc)
|
||||
|
||||
meta_kv = [
|
||||
("general.architecture", GV_STRING, "llama"),
|
||||
("general.name", GV_STRING, name),
|
||||
("llama.block_count", GV_U32, cfg.n_layer),
|
||||
("llama.embedding_length", GV_U32, cfg.n_embd),
|
||||
("llama.attention.head_count", GV_U32, cfg.n_head),
|
||||
("llama.attention.head_count_kv", GV_U32, cfg.n_head_kv),
|
||||
("llama.attention.layer_norm_rms_epsilon", GV_F32, cfg.rmsnorm_eps),
|
||||
("llama.feed_forward_length", GV_U32, cfg.ffn_dim),
|
||||
("llama.expert_count", GV_U32, cfg.n_expert),
|
||||
("llama.expert_used_count", GV_U32, cfg.n_expert_used),
|
||||
("llama.context_length", GV_U32, cfg.max_seq),
|
||||
("llama.rope.dimension_count", GV_U32, cfg.head_dim),
|
||||
("llama.rope.freq_base", GV_F32, cfg.rope_theta),
|
||||
("llama.vocab_size", GV_U32, cfg.vocab_size),
|
||||
("tokenizer.ggml.model", GV_STRING, "llama"),
|
||||
("tokenizer.ggml.pre", GV_STRING, "default"),
|
||||
]
|
||||
if tokens is not None:
|
||||
meta_kv.append(("tokenizer.ggml.tokens", GV_ARRAY, (GV_STRING, list(tokens))))
|
||||
if merges is not None:
|
||||
meta_kv.append(("tokenizer.ggml.merges", GV_ARRAY, (GV_STRING, list(merges))))
|
||||
meta_kv.append(("tokenizer.ggml.bos_token_id", GV_U32, bos))
|
||||
meta_kv.append(("tokenizer.ggml.eos_token_id", GV_U32, eos))
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(MAGIC)
|
||||
f.write(_u32(3))
|
||||
f.write(_u64(len(tensor_meta)))
|
||||
f.write(_u64(len(meta_kv)))
|
||||
for (k, vt, val) in meta_kv:
|
||||
f.write(_gstr(k))
|
||||
f.write(_u32(vt))
|
||||
f.write(_meta(vt, val))
|
||||
for (nm, dims, t, off) in tensor_meta:
|
||||
f.write(_gstr(nm))
|
||||
f.write(_u32(len(dims)))
|
||||
for d in dims:
|
||||
f.write(_u64(d))
|
||||
f.write(_u32(t))
|
||||
f.write(_u64(off))
|
||||
for chunk in data_chunks:
|
||||
f.write(chunk)
|
||||
@@ -0,0 +1,274 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
ParlzAIPlatformPAP — llama/Mixtral 架构模型 (PyTorch)。
|
||||
|
||||
与 C 引擎 (src/core/llama.c) 数学一比一：
|
||||
- pre-norm RMSNorm, 学习式绝对位置编码 -> 无；用 **RoPE** 旋转位置
|
||||
- **GQA**: n_head 查询 / n_head_kv 键值
|
||||
- **SwiGLU** FFN: down(silu(gate(x)) * up(x))
|
||||
- **Mixtral MoE**: router softmax -> top-k(used) -> 重归一化 -> 专家加权和
|
||||
激活: SiLU (x*sigmoid(x))。参数均以 (in, out) 行主序存储，C 端直接 x @ W。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Dict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
@dataclass
|
||||
class LlamaConfig:
|
||||
vocab_size: int = 32000
|
||||
n_layer: int = 4
|
||||
n_embd: int = 512
|
||||
n_head: int = 8
|
||||
n_head_kv: int = 4
|
||||
ffn_dim: int = 1024
|
||||
n_expert: int = 8
|
||||
n_expert_used: int = 2
|
||||
max_seq: int = 256
|
||||
rmsnorm_eps: float = 1e-5
|
||||
rope_theta: float = 10000.0
|
||||
|
||||
@property
|
||||
def head_dim(self) -> int:
|
||||
return self.n_embd // self.n_head
|
||||
|
||||
|
||||
def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
ms = x.pow(2).mean(dim=-1, keepdim=True)
|
||||
return w * (x * torch.rsqrt(ms + eps))
|
||||
|
||||
|
||||
def build_rope_cache(cfg: LlamaConfig, device=None):
|
||||
"""cos/sin: [max_seq, head_dim/2] (NeoX half-rotate)"""
|
||||
D = cfg.head_dim
|
||||
half = D // 2
|
||||
freqs = 1.0 / (cfg.rope_theta ** (torch.arange(0, half, dtype=torch.float32) / half))
|
||||
t = torch.arange(cfg.max_seq, dtype=torch.float32)
|
||||
angles = torch.outer(t, freqs) # [max_seq, half]
|
||||
cos = torch.cos(angles)
|
||||
sin = torch.sin(angles)
|
||||
if device is not None:
|
||||
cos, sin = cos.to(device), sin.to(device)
|
||||
return cos, sin
|
||||
|
||||
|
||||
def _apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, T: int) -> torch.Tensor:
|
||||
"""x: [B, H, T, D]（T 在 dim2）"""
|
||||
D = x.size(-1)
|
||||
x1 = x[..., :D // 2]
|
||||
x2 = x[..., D // 2:]
|
||||
c = cos[:T].unsqueeze(0).unsqueeze(1) # [1,1,T,D/2]
|
||||
s = sin[:T].unsqueeze(0).unsqueeze(1)
|
||||
out = torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], dim=-1)
|
||||
return out
|
||||
|
||||
|
||||
class LlamaBlock(nn.Module):
|
||||
def __init__(self, cfg: LlamaConfig):
|
||||
super().__init__()
|
||||
self.cfg = cfg
|
||||
H, HK, D = cfg.n_head, cfg.n_head_kv, cfg.head_dim
|
||||
self.attn_norm = nn.Parameter(torch.ones(cfg.n_embd))
|
||||
self.attn_q = nn.Parameter(torch.randn(cfg.n_embd, H * D) * 0.02) # (in,out)
|
||||
self.attn_k = nn.Parameter(torch.randn(cfg.n_embd, HK * D) * 0.02)
|
||||
self.attn_v = nn.Parameter(torch.randn(cfg.n_embd, HK * D) * 0.02)
|
||||
self.attn_o = nn.Parameter(torch.randn(cfg.n_embd, cfg.n_embd) * 0.02)
|
||||
|
||||
self.ffn_norm = nn.Parameter(torch.ones(cfg.n_embd))
|
||||
if cfg.n_expert > 0:
|
||||
self.ffn_gate_inp = nn.Parameter(torch.randn(cfg.n_embd, cfg.n_expert) * 0.02) # router
|
||||
self.ffn_experts = nn.ModuleList()
|
||||
for _ in range(cfg.n_expert):
|
||||
e = nn.Module()
|
||||
e.ffn_gate = nn.Parameter(torch.randn(cfg.n_embd, cfg.ffn_dim) * 0.02)
|
||||
e.ffn_up = nn.Parameter(torch.randn(cfg.n_embd, cfg.ffn_dim) * 0.02)
|
||||
e.ffn_down = nn.Parameter(torch.randn(cfg.ffn_dim, cfg.n_embd) * 0.02)
|
||||
self.ffn_experts.append(e)
|
||||
else:
|
||||
self.ffn_gate = nn.Parameter(torch.randn(cfg.n_embd, cfg.ffn_dim) * 0.02)
|
||||
self.ffn_up = nn.Parameter(torch.randn(cfg.n_embd, cfg.ffn_dim) * 0.02)
|
||||
self.ffn_down = nn.Parameter(torch.randn(cfg.ffn_dim, cfg.n_embd) * 0.02)
|
||||
|
||||
def forward(self, x, cos, sin):
|
||||
cfg = self.cfg
|
||||
B, T, C = x.shape
|
||||
H, HK, D = cfg.n_head, cfg.n_head_kv, cfg.head_dim
|
||||
hk = H // HK # 每组 kv 对应几个查询头
|
||||
|
||||
a = _rmsnorm(x, self.attn_norm, cfg.rmsnorm_eps)
|
||||
q = a @ self.attn_q # [B,T,H*D]
|
||||
k = a @ self.attn_k
|
||||
v = a @ self.attn_v
|
||||
q = q.view(B, T, H, D).transpose(1, 2) # [B,H,T,D]
|
||||
k = k.view(B, T, HK, D).transpose(1, 2)
|
||||
v = v.view(B, T, HK, D).transpose(1, 2)
|
||||
q = _apply_rope(q, cos, sin, T)
|
||||
k = _apply_rope(k, cos, sin, T)
|
||||
|
||||
weight = torch.zeros(B, H, T, T, device=x.device)
|
||||
for h in range(H):
|
||||
kvh = h // hk
|
||||
qh = q[:, h] # [B,T,D]
|
||||
kh = k[:, kvh] # [B,T,D]
|
||||
sc = (qh @ kh.transpose(-2, -1)) / math.sqrt(D)
|
||||
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
|
||||
sc = sc.masked_fill(~mask, float("-inf"))
|
||||
w = F.softmax(sc, dim=-1)
|
||||
weight[:, h] = w
|
||||
# 聚合
|
||||
att = torch.zeros(B, T, H * D, device=x.device)
|
||||
for h in range(H):
|
||||
kvh = h // hk
|
||||
vh = v[:, kvh] # [B,T,D]
|
||||
o = weight[:, h] @ vh # [B,T,D]
|
||||
att[:, :, h * D:(h + 1) * D] = o
|
||||
att = att.view(B, T, C)
|
||||
x = x + att @ self.attn_o
|
||||
|
||||
f = _rmsnorm(x, self.ffn_norm, cfg.rmsnorm_eps)
|
||||
if cfg.n_expert > 0:
|
||||
logits = f @ self.ffn_gate_inp # [B,T,n_expert]
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
top = torch.topk(probs, k=cfg.n_expert_used, dim=-1)
|
||||
wtop = top.values # [B,T,used]
|
||||
idx = top.indices
|
||||
wtop = wtop / (wtop.sum(dim=-1, keepdim=True) + 1e-9) # 重归一化
|
||||
N = B * T
|
||||
out = torch.zeros(N, C, device=x.device)
|
||||
f2 = f.reshape(N, C)
|
||||
for t in range(cfg.n_expert_used):
|
||||
eidx = idx[..., t].reshape(-1)
|
||||
w = wtop[..., t].reshape(-1, 1)
|
||||
for e in range(cfg.n_expert):
|
||||
sel = (eidx == e)
|
||||
if sel.any():
|
||||
exp = self.ffn_experts[e]
|
||||
z = f2[sel] @ exp.ffn_gate
|
||||
z = F.silu(z) * (f2[sel] @ exp.ffn_up)
|
||||
out[sel] += w[sel] * (z @ exp.ffn_down)
|
||||
x = x + out.view(B, T, C)
|
||||
else:
|
||||
g = F.silu(f @ self.ffn_gate)
|
||||
u = f @ self.ffn_up
|
||||
up = g * u
|
||||
x = x + (up @ self.ffn_down)
|
||||
return x
|
||||
|
||||
|
||||
class LlamaMoE(nn.Module):
|
||||
def __init__(self, cfg: LlamaConfig):
|
||||
super().__init__()
|
||||
self.cfg = cfg
|
||||
self.token_embd = nn.Parameter(torch.randn(cfg.vocab_size, cfg.n_embd) * 0.02)
|
||||
self.blocks = nn.ModuleList([LlamaBlock(cfg) for _ in range(cfg.n_layer)])
|
||||
self.output_norm = nn.Parameter(torch.ones(cfg.n_embd))
|
||||
self.output = nn.Parameter(torch.randn(cfg.n_embd, cfg.vocab_size) * 0.02)
|
||||
self._cos = None
|
||||
self._sin = None
|
||||
|
||||
def _rope(self):
|
||||
if self._cos is None:
|
||||
c, s = build_rope_cache(self.cfg)
|
||||
self._cos, self._sin = c, s
|
||||
return self._cos, self._sin
|
||||
|
||||
def forward(self, idx):
|
||||
T = idx.size(1)
|
||||
x = self.token_embd[idx] # [B,T,C]
|
||||
cos, sin = self._rope()
|
||||
for blk in self.blocks:
|
||||
x = blk(x, cos, sin)
|
||||
x = _rmsnorm(x, self.output_norm, self.cfg.rmsnorm_eps)
|
||||
logits = x @ self.output
|
||||
return logits
|
||||
|
||||
@torch.no_grad()
|
||||
def generate_from_ids(self, init_ids, max_new, temperature=0.8, top_k=40):
|
||||
device = next(self.parameters()).device
|
||||
self.to(device)
|
||||
x = torch.tensor([init_ids], dtype=torch.long, device=device)
|
||||
out = list(init_ids)
|
||||
for _ in range(max_new):
|
||||
logits = self.forward(x[:, -self.cfg.max_seq:])[:, -1, :]
|
||||
if temperature <= 0:
|
||||
nxt = torch.argmax(logits).item()
|
||||
else:
|
||||
if temperature > 0: logits = logits / temperature
|
||||
if top_k > 0:
|
||||
v, _ = torch.topk(logits, top_k)
|
||||
logits[logits < v[-1]] = float("-inf")
|
||||
nxt = torch.multinomial(F.softmax(logits, dim=-1), 1).item()
|
||||
out.append(nxt)
|
||||
x = torch.cat([x, torch.tensor([[nxt]], device=device)], dim=1)
|
||||
return out
|
||||
|
||||
|
||||
def export_tensors(model: LlamaMoE) -> Dict[str, torch.Tensor]:
|
||||
"""收集为 GGUF llama 命名的 (in,out) 张量。"""
|
||||
cfg = model.cfg
|
||||
tens: Dict[str, torch.Tensor] = {}
|
||||
tens["token_embd.weight"] = model.token_embd.detach()
|
||||
for i, b in enumerate(model.blocks):
|
||||
p = f"blk.{i}"
|
||||
tens[f"{p}.attn_norm.weight"] = b.attn_norm.detach()
|
||||
tens[f"{p}.attn_q.weight"] = b.attn_q.detach()
|
||||
tens[f"{p}.attn_k.weight"] = b.attn_k.detach()
|
||||
tens[f"{p}.attn_v.weight"] = b.attn_v.detach()
|
||||
tens[f"{p}.attn_o.weight"] = b.attn_o.detach()
|
||||
tens[f"{p}.ffn_norm.weight"] = b.ffn_norm.detach()
|
||||
if cfg.n_expert > 0:
|
||||
tens[f"{p}.ffn_gate_inp.weight"] = b.ffn_gate_inp.detach()
|
||||
for e, ex in enumerate(b.ffn_experts):
|
||||
tens[f"{p}.ffn_experts.{e}.ffn_gate.weight"] = ex.ffn_gate.detach()
|
||||
tens[f"{p}.ffn_experts.{e}.ffn_up.weight"] = ex.ffn_up.detach()
|
||||
tens[f"{p}.ffn_experts.{e}.ffn_down.weight"] = ex.ffn_down.detach()
|
||||
else:
|
||||
tens[f"{p}.ffn_gate.weight"] = b.ffn_gate.detach()
|
||||
tens[f"{p}.ffn_up.weight"] = b.ffn_up.detach()
|
||||
tens[f"{p}.ffn_down.weight"] = b.ffn_down.detach()
|
||||
tens["output_norm.weight"] = model.output_norm.detach()
|
||||
tens["output.weight"] = model.output.detach()
|
||||
return tens
|
||||
|
||||
|
||||
def load_llama_from_gguf(path: str, cfg: LlamaConfig) -> LlamaMoE:
|
||||
"""从 GGUF 张量重建 LlamaMoE（用于 C/torch 对拍）。"""
|
||||
import gguf_format
|
||||
tens, meta, ver = gguf_format.read_gguf(path)
|
||||
model = LlamaMoE(cfg)
|
||||
sd: Dict[str, torch.Tensor] = {}
|
||||
|
||||
def T(k):
|
||||
return torch.from_numpy(tens[k])
|
||||
|
||||
sd["token_embd"] = T("token_embd.weight")
|
||||
sd["output_norm"] = T("output_norm.weight").reshape(-1)
|
||||
sd["output"] = T("output.weight")
|
||||
for i in range(cfg.n_layer):
|
||||
b = f"blocks.{i}"
|
||||
g = f"blk.{i}"
|
||||
sd[f"{b}.attn_norm"] = T(f"{g}.attn_norm.weight").reshape(-1)
|
||||
sd[f"{b}.attn_q"] = T(f"{g}.attn_q.weight")
|
||||
sd[f"{b}.attn_k"] = T(f"{g}.attn_k.weight")
|
||||
sd[f"{b}.attn_v"] = T(f"{g}.attn_v.weight")
|
||||
sd[f"{b}.attn_o"] = T(f"{g}.attn_o.weight")
|
||||
sd[f"{b}.ffn_norm"] = T(f"{g}.ffn_norm.weight").reshape(-1)
|
||||
if cfg.n_expert > 0:
|
||||
sd[f"{b}.ffn_gate_inp"] = T(f"{g}.ffn_gate_inp.weight")
|
||||
for e in range(cfg.n_expert):
|
||||
sd[f"{b}.ffn_experts.{e}.ffn_gate"] = T(f"{g}.ffn_experts.{e}.ffn_gate.weight")
|
||||
sd[f"{b}.ffn_experts.{e}.ffn_up"] = T(f"{g}.ffn_experts.{e}.ffn_up.weight")
|
||||
sd[f"{b}.ffn_experts.{e}.ffn_down"] = T(f"{g}.ffn_experts.{e}.ffn_down.weight")
|
||||
else:
|
||||
sd[f"{b}.ffn_gate"] = T(f"{g}.ffn_gate.weight")
|
||||
sd[f"{b}.ffn_up"] = T(f"{g}.ffn_up.weight")
|
||||
sd[f"{b}.ffn_down"] = T(f"{g}.ffn_down.weight")
|
||||
model.load_state_dict(sd, strict=True)
|
||||
model.eval()
|
||||
return model
|
||||
@@ -0,0 +1,73 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""GPT-2 / llama 的 byte-level BPE 分词器（读 GGUF tokenizer.ggml.tokens/merges）。"""
|
||||
from __future__ import annotations
|
||||
from typing import List, Tuple
|
||||
|
||||
|
||||
def byte_to_unicode() -> dict:
|
||||
"""GPT-2 的 byte→unicode 映射（与 llama.cpp’llama’ tokenizer 一致）。"""
|
||||
bs = (list(range(ord("!"), ord("~") + 1)) +
|
||||
list(range(ord("\u00a1"), ord("\u00ac") + 1)) +
|
||||
list(range(ord("\u00ae"), ord("\u00ff") + 1)))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(256):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(256 + n)
|
||||
n += 1
|
||||
return dict(zip([b for b in bs], [chr(c) for c in cs]))
|
||||
|
||||
|
||||
def unicode_to_byte() -> dict:
|
||||
return {v: k for k, v in byte_to_unicode().items()}
|
||||
|
||||
|
||||
class LlamaTokenizer:
|
||||
def __init__(self, tokens: List[str], merges: List[str]):
|
||||
self.tokens = tokens
|
||||
# 字符 -> token id（只收录单字符 token）
|
||||
self.symbol2id = {t: i for i, t in enumerate(tokens) if len(t) == 1}
|
||||
# merges: "a b" -> rank
|
||||
self.ranks = {}
|
||||
self.pairs = []
|
||||
for i, m in enumerate(merges):
|
||||
a, b = m.split(" ", 1) if " " in m else (m, "")
|
||||
self.ranks[(a, b)] = i
|
||||
self.pairs.append((a, b))
|
||||
|
||||
def encode(self, text: str, bos: bool = False, eos: bool = False,
|
||||
bos_id: int = 151646, eos_id: int = 151643) -> List[int]:
|
||||
b2u = byte_to_unicode()
|
||||
syms = [b2u[b] for b in text.encode("utf-8")]
|
||||
# BPE 合并（最小 rank）
|
||||
while len(syms) >= 2:
|
||||
best = None
|
||||
for i in range(len(syms) - 1):
|
||||
r = self.ranks.get((syms[i], syms[i + 1]))
|
||||
if r is not None and (best is None or r < best[0]):
|
||||
best = (r, i)
|
||||
if best is None:
|
||||
break
|
||||
_, i = best
|
||||
syms[i] = syms[i] + syms[i + 1]
|
||||
del syms[i + 1]
|
||||
idmap = {t: i for i, t in enumerate(self.tokens)}
|
||||
out = [idmap.get(s, self.symbol2id.get(s, 0)) for s in syms]
|
||||
if bos:
|
||||
out = [bos_id] + out
|
||||
if eos:
|
||||
out = out + [eos_id]
|
||||
return out
|
||||
|
||||
def decode(self, ids: List[int]) -> str:
|
||||
u2b = unicode_to_byte()
|
||||
chars = []
|
||||
for i in ids:
|
||||
if 0 <= i < len(self.tokens):
|
||||
chars.append(self.tokens[i])
|
||||
s = "".join(chars)
|
||||
out = bytearray()
|
||||
for ch in s:
|
||||
out.append(u2b.get(ch, ord(ch) & 0xFF))
|
||||
return out.decode("utf-8", errors="replace")
|
||||
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""生成一份多轮闲聊对话语料（User:/Assistant: 格式），用于训练聊天模型。"""
|
||||
import random
|
||||
|
||||
|
||||
def build_pools():
|
||||
P = {}
|
||||
P["names"] = ["Alice", "Bob", "Sam", "Luna", "Max", "Emma", "Kai", "Nora", "Leo", "Mia"]
|
||||
P["hobbies"] = ["reading", "hiking", "cooking", "painting", "chess", "yoga", "photography", "fishing", "gaming", "gardening"]
|
||||
P["foods"] = ["pizza", "sushi", "ramen", "tacos", "pasta", "salad", "biryani", "dumplings", "curry", "pancakes"]
|
||||
P["cities"] = ["Tokyo", "Paris", "Berlin", "Lisbon", "Seoul", "Toronto", "Sydney", "Rome", "Kyoto", "Prague"]
|
||||
P["weather"] = ["sunny", "rainy", "cloudy", "snowy", "windy", "clear"]
|
||||
P["movies"] = ["sci-fi", "comedy", "drama", "horror", "adventure", "romance"]
|
||||
P["music"] = ["pop", "jazz", "rock", "classical", "hip-hop", "folk"]
|
||||
P["ports"] = ["pets", "cats", "dogs", "birds", "fish"]
|
||||
return P
|
||||
|
||||
|
||||
def gen_conversation(P, rng):
|
||||
n_turns = rng.randint(3, 6)
|
||||
turns = []
|
||||
for i in range(n_turns):
|
||||
if i % 2 == 0: # user
|
||||
r = rng.random()
|
||||
if r < 0.20:
|
||||
u = rng.choice(["Hi", "Hello", "Hey", "Good morning", "What's up", "Hi there"])
|
||||
elif r < 0.35:
|
||||
u = f"How are you doing" + ("?" if rng.random() < 0.7 else "")
|
||||
elif r < 0.5:
|
||||
u = f"I like {rng.choice(P['hobbies'])}. What about you?"
|
||||
elif r < 0.62:
|
||||
u = f"Have you been to {rng.choice(P['cities'])}?"
|
||||
elif r < 0.72:
|
||||
u = f"What kind of {rng.choice(['food', 'music', 'movies'])} do you like?"
|
||||
elif r < 0.8:
|
||||
u = f"Do you have any {rng.choice(P['ports'])}?"
|
||||
elif r < 0.9:
|
||||
u = f"Can you tell me a {rng.choice(['joke', 'fact', 'story'])}?"
|
||||
else:
|
||||
u = "That's nice. Tell me more."
|
||||
turns.append(("User", u))
|
||||
else: # assistant
|
||||
r = rng.random()
|
||||
if r < 0.2:
|
||||
a = rng.choice(["Hi! How can I help you today?", "Hello! It's good to hear from you.", "Hey there! What's on your mind?"])
|
||||
elif r < 0.35:
|
||||
a = rng.choice(["I'm doing great, thanks for asking!", "Pretty good! How about you?", "Can't complain. How are you?"])
|
||||
elif r < 0.5:
|
||||
a = f"That sounds fun! I enjoy {rng.choice(P['hobbies'])} too."
|
||||
elif r < 0.62:
|
||||
a = f"I haven't been to {rng.choice(P['cities'])}, but I've heard it's lovely."
|
||||
elif r < 0.72:
|
||||
a = f"I'd say I like {rng.choice(P['music'])} and {rng.choice(P['movies'])}. You?"
|
||||
elif r < 0.8:
|
||||
a = rng.choice(["I do! I really like dogs and cats.", "I think pets are wonderful companions."])
|
||||
elif r < 0.9:
|
||||
a = rng.choice(["Here's one: why did the programmer quit? Because he didn't get arrays.", "Did you know honey never spoils?", "Let me tell you a short story about a curious little robot."])
|
||||
else:
|
||||
a = rng.choice(["That makes sense.", "I see what you mean.", "Interesting point!", "Thanks for sharing that."])
|
||||
turns.append(("Assistant", a))
|
||||
return turns
|
||||
|
||||
|
||||
def main(out="data/chat.txt", n=4000, seed=7):
|
||||
random.seed(seed)
|
||||
P = build_pools()
|
||||
lines = []
|
||||
for _ in range(n):
|
||||
for role, text in gen_conversation(P, random):
|
||||
lines.append(f"{role}: {text}")
|
||||
lines.append("") # 空行分隔对话
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
print(f"chat corpus -> {out} : {sum(len(l) for l in lines)} chars, {n} conversations")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
main(out=sys.argv[1] if len(sys.argv) > 1 else "data/chat.txt")
|
||||
@@ -0,0 +1,40 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""生成一个随机小模型并导出 .pap，用于 C 与 torch 的对拍验证（不训练）。"""
|
||||
import os
|
||||
|
||||
import model as model_mod
|
||||
import pap_format
|
||||
import tokenizer as tok
|
||||
|
||||
|
||||
def main(out: str = "/mnt/f/ParlzMAI/models/_smoke.pap"):
|
||||
# 纯字节词表（num_merges=0），最干净
|
||||
vocab = [bytes([b]) for b in range(256)]
|
||||
merges = []
|
||||
cfg = {"vocab_size": 256, "n_layer": 2, "d_model": 64, "n_head": 4,
|
||||
"d_ff": 128, "moe_n_experts": 4, "moe_top_k": 2, "d_expert": 128,
|
||||
"max_seq_len": 32, "rmsnorm_eps": 1e-5, "moe_mask": [1, 0]}
|
||||
model = model_mod.ParlzGPTMoE(cfg)
|
||||
|
||||
tens = model_mod.export_tensors(model)
|
||||
pcfg = pap_format.ModelConfig()
|
||||
pcfg.vocab_size = cfg["vocab_size"]
|
||||
pcfg.n_layer = cfg["n_layer"]
|
||||
pcfg.d_model = cfg["d_model"]
|
||||
pcfg.n_head = cfg["n_head"]
|
||||
pcfg.d_ff = cfg["d_ff"]
|
||||
pcfg.moe_n_experts = cfg["moe_n_experts"]
|
||||
pcfg.moe_top_k = cfg["moe_top_k"]
|
||||
pcfg.d_expert = cfg["d_expert"]
|
||||
pcfg.max_seq_len = cfg["max_seq_len"]
|
||||
pcfg.num_merges = len(merges)
|
||||
pcfg.tie_weights = 0
|
||||
pcfg.rmsnorm_eps = cfg["rmsnorm_eps"]
|
||||
pcfg.moe_mask = list(cfg["moe_mask"])
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
pap_format.save_pap(out, pcfg, vocab, merges, tens)
|
||||
print(f"exported random model -> {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
ParlzAIPlatformPAP — GPT + MoE 模型 (PyTorch)。
|
||||
|
||||
与 C 引擎 (src/core/transformer.c) 数学一对一:
|
||||
- pre-norm transformer, RMSNorm
|
||||
- 学习式绝对位置编码 (无 RoPE)
|
||||
- 缩放点积注意力 (无 GQA), 因果掩码
|
||||
- 每个 block: norm1 -> attn -> +res, norm2 -> ffn/moe -> +res
|
||||
- FFN 层可选 MoE: router 线性 + softmax -> top-k -> 专家加权和
|
||||
- 激活: tanh 近似 GELU (与 C 的 mo_gelu 一致)
|
||||
|
||||
导出 .pap 时所有 nn.Linear 权重做 .t() 转置成 (in,out)，C 端直接 x @ W。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Dict, List
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
"""y = x / sqrt(mean(x^2)+eps) * w，沿最后一维。"""
|
||||
mean_sq = x.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(mean_sq + eps)
|
||||
return w * x
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int, eps: float = 1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(dim))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, x):
|
||||
return rmsnorm(x, self.weight, self.eps)
|
||||
|
||||
|
||||
class CausalSelfAttention(nn.Module):
|
||||
def __init__(self, d_model: int, n_head: int):
|
||||
super().__init__()
|
||||
assert d_model % n_head == 0
|
||||
self.n_head = n_head
|
||||
self.head_dim = d_model // n_head
|
||||
self.d_model = d_model
|
||||
self.wq = nn.Linear(d_model, d_model, bias=False)
|
||||
self.wk = nn.Linear(d_model, d_model, bias=False)
|
||||
self.wv = nn.Linear(d_model, d_model, bias=False)
|
||||
self.wo = nn.Linear(d_model, d_model, bias=False)
|
||||
|
||||
def forward(self, x):
|
||||
B, T, C = x.shape
|
||||
q = self.wq(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
|
||||
k = self.wk(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
|
||||
v = self.wv(x).view(B, T, self.n_head, self.head_dim).transpose(1, 2)
|
||||
att = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
|
||||
mask = torch.tril(torch.ones(T, T, device=x.device, dtype=torch.bool))
|
||||
att = att.masked_fill(~mask, float("-inf"))
|
||||
att = F.softmax(att, dim=-1)
|
||||
y = att @ v
|
||||
y = y.transpose(1, 2).contiguous().view(B, T, C)
|
||||
return self.wo(y)
|
||||
|
||||
|
||||
class MoEExpert(nn.Module):
|
||||
def __init__(self, d_model: int, d_expert: int):
|
||||
super().__init__()
|
||||
self.w1 = nn.Linear(d_model, d_expert, bias=False) # 上投影
|
||||
self.w2 = nn.Linear(d_expert, d_model, bias=False) # 下投影
|
||||
|
||||
def forward(self, x):
|
||||
return self.w2(F.gelu(self.w1(x), approximate="tanh"))
|
||||
|
||||
|
||||
class MoELayer(nn.Module):
|
||||
def __init__(self, d_model: int, n_experts: int, top_k: int, d_expert: int):
|
||||
super().__init__()
|
||||
self.n_experts = n_experts
|
||||
self.top_k = top_k
|
||||
self.router = nn.Linear(d_model, n_experts, bias=False)
|
||||
self.experts = nn.ModuleList([MoEExpert(d_model, d_expert) for _ in range(n_experts)])
|
||||
|
||||
def forward(self, x):
|
||||
B, T, C = x.shape
|
||||
logits = self.router(x) # (B,T,n_experts)
|
||||
probs = F.softmax(logits, dim=-1) # 路由概率
|
||||
topk = torch.topk(probs, k=self.top_k, dim=-1) # values, indices
|
||||
sel_w = topk.values # (B,T,top_k)
|
||||
sel_i = topk.indices # (B,T,top_k)
|
||||
out = torch.zeros_like(x)
|
||||
for t in range(self.top_k):
|
||||
e_idx = sel_i[..., t].reshape(-1) # (B*T,)
|
||||
wt = sel_w[..., t].reshape(-1, 1) # (B*T,1)
|
||||
src = x.reshape(B * T, C)
|
||||
# 用 gather 取每行对应专家的输出
|
||||
exp_out = torch.zeros(B * T, C, device=x.device)
|
||||
for e in range(self.n_experts):
|
||||
sel = (e_idx == e)
|
||||
if sel.any():
|
||||
sub = self.experts[e](src[sel])
|
||||
exp_out[sel] = sub
|
||||
out += (wt * exp_out).view(B, T, C)
|
||||
return out
|
||||
|
||||
|
||||
class Block(nn.Module):
|
||||
def __init__(self, d_model: int, n_head: int, cfg, is_moe: bool):
|
||||
super().__init__()
|
||||
self.is_moe = is_moe
|
||||
self.ln1 = RMSNorm(d_model, cfg["rmsnorm_eps"])
|
||||
self.attn = CausalSelfAttention(d_model, n_head)
|
||||
self.ln2 = RMSNorm(d_model, cfg["rmsnorm_eps"])
|
||||
if is_moe:
|
||||
self.moe = MoELayer(d_model, cfg["moe_n_experts"], cfg["moe_top_k"], cfg["d_expert"])
|
||||
else:
|
||||
self.ffn = nn.Sequential(
|
||||
nn.Linear(d_model, cfg["d_ff"], bias=False),
|
||||
nn.GELU(approximate="tanh"),
|
||||
nn.Linear(cfg["d_ff"], d_model, bias=False),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x = x + self.attn(self.ln1(x))
|
||||
if self.is_moe:
|
||||
x = x + self.moe(self.ln2(x))
|
||||
else:
|
||||
x = x + self.ffn(self.ln2(x))
|
||||
return x
|
||||
|
||||
|
||||
class ParlzGPTMoE(nn.Module):
|
||||
def __init__(self, cfg):
|
||||
super().__init__()
|
||||
self.cfg = cfg
|
||||
self.vocab_size = cfg["vocab_size"]
|
||||
self.max_seq_len = cfg["max_seq_len"]
|
||||
self.wte = nn.Embedding(cfg["vocab_size"], cfg["d_model"])
|
||||
self.wpe = nn.Embedding(cfg["max_seq_len"], cfg["d_model"])
|
||||
self.drop = nn.Dropout(0.0)
|
||||
self.blocks = nn.ModuleList([
|
||||
Block(cfg["d_model"], cfg["n_head"], cfg, cfg["moe_mask"][l])
|
||||
for l in range(cfg["n_layer"])
|
||||
])
|
||||
self.ln_f = RMSNorm(cfg["d_model"], cfg["rmsnorm_eps"])
|
||||
self.lm_head = nn.Linear(cfg["d_model"], cfg["vocab_size"], bias=False)
|
||||
|
||||
def forward(self, idx, targets=None):
|
||||
T = idx.size(1)
|
||||
assert T <= self.max_seq_len
|
||||
pos = torch.arange(0, T, device=idx.device).unsqueeze(0)
|
||||
x = self.wte(idx) + self.wpe(pos)
|
||||
x = self.drop(x)
|
||||
for blk in self.blocks:
|
||||
x = blk(x)
|
||||
x = self.ln_f(x)
|
||||
logits = self.lm_head(x)
|
||||
if targets is not None:
|
||||
loss = F.cross_entropy(logits.view(-1, self.vocab_size), targets.view(-1))
|
||||
return logits, loss
|
||||
return logits, None
|
||||
|
||||
@torch.no_grad()
|
||||
def generate_from_ids(self, init_ids, max_new, temperature=0.8, top_k=40):
|
||||
"""自回归采样生成（训练途中示例用，无需 tokenizer）。返回 id 列表。"""
|
||||
device = next(self.parameters()).device
|
||||
x = torch.tensor([init_ids], dtype=torch.long, device=device)
|
||||
out = list(init_ids)
|
||||
for _ in range(max_new):
|
||||
logits, _ = self(x[:, -self.max_seq_len:])
|
||||
logits = logits[0, -1]
|
||||
if temperature <= 0:
|
||||
nxt = torch.argmax(logits).item()
|
||||
out.append(nxt)
|
||||
x = torch.cat([x, torch.tensor([[nxt]], device=device)], dim=1)
|
||||
continue
|
||||
if temperature > 0:
|
||||
logits = logits / temperature
|
||||
if top_k > 0:
|
||||
v, _ = torch.topk(logits, top_k)
|
||||
logits[logits < v[-1]] = float("-inf")
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
nxt = torch.multinomial(probs, 1).item()
|
||||
out.append(nxt)
|
||||
x = torch.cat([x, torch.tensor([[nxt]], device=device)], dim=1)
|
||||
return out
|
||||
|
||||
|
||||
def export_tensors(model: "ParlzGPTMoE"):
|
||||
"""收集权重为 .pap 命名张量；Linear 权重转置为 (in,out) 行主序。"""
|
||||
tens: Dict[str, torch.Tensor] = {}
|
||||
tens["wte"] = model.wte.weight.detach() # (vocab,d)
|
||||
tens["wpe"] = model.wpe.weight.detach() # (max_seq,d)
|
||||
for l, blk in enumerate(model.blocks):
|
||||
p = f"l{l}"
|
||||
tens[f"{p}.norm1"] = blk.ln1.weight.detach()
|
||||
tens[f"{p}.attn.wq"] = blk.attn.wq.weight.detach().t()
|
||||
tens[f"{p}.attn.wk"] = blk.attn.wk.weight.detach().t()
|
||||
tens[f"{p}.attn.wv"] = blk.attn.wv.weight.detach().t()
|
||||
tens[f"{p}.attn.wo"] = blk.attn.wo.weight.detach().t()
|
||||
tens[f"{p}.norm2"] = blk.ln2.weight.detach()
|
||||
if blk.is_moe:
|
||||
tens[f"{p}.moe.router"] = blk.moe.router.weight.detach().t()
|
||||
for e, expert in enumerate(blk.moe.experts):
|
||||
tens[f"{p}.moe.exp{e}.w1"] = expert.w1.weight.detach().t()
|
||||
tens[f"{p}.moe.exp{e}.w2"] = expert.w2.weight.detach().t()
|
||||
else:
|
||||
tens[f"{p}.ffn.w1"] = blk.ffn[0].weight.detach().t()
|
||||
tens[f"{p}.ffn.w2"] = blk.ffn[2].weight.detach().t()
|
||||
tens["norm_final"] = model.ln_f.weight.detach()
|
||||
tens["lm_head"] = model.lm_head.weight.detach().t() # (vocab,d)->(d,vocab)? no
|
||||
# 注意: lm_head 是 (vocab,d)，C 端 h @ lm_head 需 (d,vocab)。torch Linear 权重是 (vocab,d)，
|
||||
# 转置应为 (d,vocab)。但 export 里我们要求 (in,out)；lm_head 的 "输入"是 d，"输出"是 vocab。
|
||||
tens["lm_head"] = model.lm_head.weight.detach().t() # 保证 (d_model, vocab_size)? 见下
|
||||
return tens
|
||||
|
||||
|
||||
def load_pap_model(path: str):
|
||||
"""从 .pap 重建 torch 模型（反推 export_tensors 的转置）。返回 (model, vocab, merges)。"""
|
||||
import pap_format
|
||||
|
||||
cfgp, vocab, merges, tens = pap_format.read_pap(path)
|
||||
cfg = {
|
||||
"vocab_size": cfgp.vocab_size,
|
||||
"n_layer": cfgp.n_layer,
|
||||
"d_model": cfgp.d_model,
|
||||
"n_head": cfgp.n_head,
|
||||
"d_ff": cfgp.d_ff,
|
||||
"moe_n_experts": cfgp.moe_n_experts,
|
||||
"moe_top_k": cfgp.moe_top_k,
|
||||
"d_expert": cfgp.d_expert,
|
||||
"max_seq_len": cfgp.max_seq_len,
|
||||
"rmsnorm_eps": cfgp.rmsnorm_eps,
|
||||
"moe_mask": list(cfgp.moe_mask),
|
||||
}
|
||||
model = ParlzGPTMoE(cfg)
|
||||
sd = {}
|
||||
sd["wte.weight"] = torch.from_numpy(tens["wte"])
|
||||
sd["wpe.weight"] = torch.from_numpy(tens["wpe"])
|
||||
for l in range(cfg["n_layer"]):
|
||||
sd[f"blocks.{l}.ln1.weight"] = torch.from_numpy(tens[f"l{l}.norm1"]).reshape(-1)
|
||||
sd[f"blocks.{l}.attn.wq.weight"] = torch.from_numpy(tens[f"l{l}.attn.wq"].T)
|
||||
sd[f"blocks.{l}.attn.wk.weight"] = torch.from_numpy(tens[f"l{l}.attn.wk"].T)
|
||||
sd[f"blocks.{l}.attn.wv.weight"] = torch.from_numpy(tens[f"l{l}.attn.wv"].T)
|
||||
sd[f"blocks.{l}.attn.wo.weight"] = torch.from_numpy(tens[f"l{l}.attn.wo"].T)
|
||||
sd[f"blocks.{l}.ln2.weight"] = torch.from_numpy(tens[f"l{l}.norm2"]).reshape(-1)
|
||||
if cfg["moe_mask"][l]:
|
||||
sd[f"blocks.{l}.moe.router.weight"] = torch.from_numpy(tens[f"l{l}.moe.router"].T)
|
||||
for e in range(cfg["moe_n_experts"]):
|
||||
sd[f"blocks.{l}.moe.experts.{e}.w1.weight"] = torch.from_numpy(tens[f"l{l}.moe.exp{e}.w1"].T)
|
||||
sd[f"blocks.{l}.moe.experts.{e}.w2.weight"] = torch.from_numpy(tens[f"l{l}.moe.exp{e}.w2"].T)
|
||||
else:
|
||||
sd[f"blocks.{l}.ffn.0.weight"] = torch.from_numpy(tens[f"l{l}.ffn.w1"].T)
|
||||
sd[f"blocks.{l}.ffn.2.weight"] = torch.from_numpy(tens[f"l{l}.ffn.w2"].T)
|
||||
sd["ln_f.weight"] = torch.from_numpy(tens["norm_final"]).reshape(-1)
|
||||
sd["lm_head.weight"] = torch.from_numpy(tens["lm_head"].T)
|
||||
model.load_state_dict(sd, strict=True)
|
||||
model.eval()
|
||||
return model, vocab, merges
|
||||
@@ -0,0 +1,293 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
ParlzAIPlatformPAP — .pap 模型文件格式 (version 1)
|
||||
|
||||
二进制布局 (little-endian):
|
||||
|
||||
off size 字段
|
||||
0 4 magic = b"PAP1"
|
||||
4 4 version (u32) = 1
|
||||
8 4 dtype (u32) = 0 # 0=f32, 1=Q8(未实现)
|
||||
12 4 vocab_size (u32)
|
||||
16 4 n_layer (u32)
|
||||
20 4 d_model (u32)
|
||||
24 4 n_head (u32)
|
||||
28 4 d_ff (u32) # 稠密 FFN 中间层
|
||||
32 4 moe_n_experts (u32)
|
||||
36 4 moe_top_k (u32)
|
||||
40 4 d_expert (u32) # MoE 专家中间层
|
||||
44 4 max_seq_len (u32)
|
||||
48 4 num_merges (u32)
|
||||
52 4 n_special (u32)
|
||||
56 1 tie_weights (u8)
|
||||
57 1 reserved
|
||||
58 1 reserved
|
||||
59 1 reserved
|
||||
60 4 rmsnorm_eps (f32)
|
||||
64 n moe_mask: n_layer bytes (1=MoE, 0=dense FFN)
|
||||
|
||||
--- vocab 段 ---
|
||||
u32 vocab_blob_len
|
||||
blob: for id in 0..vocab_size-1: u32 len + len bytes
|
||||
for m in 0..num_merges-1 : u32 a + u32 b
|
||||
|
||||
--- weights 段 ---
|
||||
u32 n_tensors
|
||||
for each: u32 name_len + name + u32 rows + u32 cols + rows*cols f32(row-major)
|
||||
|
||||
张量命名（行主序，列为 (in, out)，C 端直接 x @ W）：
|
||||
wte (vocab,d) wpe (max_seq,d)
|
||||
l{L}.norm1, l{L}.attn.{wq,wk,wv,wo}, l{L}.norm2
|
||||
l{L}.moe.router (d,n_experts) ; l{L}.moe.exp{E}.{w1(d,d_expert),w2(d_expert,d)}
|
||||
l{L}.ffn.{w1(d,d_ff),w2(d_ff,d)} (稠密层)
|
||||
norm_final (d) ; lm_head (vocab,d)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
PARZ_MAGIC = b"PAP1"
|
||||
VERSION = 1
|
||||
|
||||
_M = "<" # little-endian
|
||||
|
||||
|
||||
def _u32(v: int) -> bytes:
|
||||
return struct.pack(_M + "I", int(v) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def _f32(v: float) -> bytes:
|
||||
return struct.pack(_M + "f", float(v))
|
||||
|
||||
|
||||
class ModelConfig:
|
||||
def __init__(self):
|
||||
self.vocab_size = 0
|
||||
self.n_layer = 0
|
||||
self.d_model = 0
|
||||
self.n_head = 0
|
||||
self.d_ff = 0
|
||||
self.moe_n_experts = 0
|
||||
self.moe_top_k = 0
|
||||
self.d_expert = 0
|
||||
self.max_seq_len = 0
|
||||
self.num_merges = 0
|
||||
self.n_special = 0
|
||||
self.tie_weights = 0
|
||||
self.rmsnorm_eps = 1e-5
|
||||
self.moe_mask: List[int] = []
|
||||
|
||||
@property
|
||||
def head_dim(self) -> int:
|
||||
return self.d_model // self.n_head
|
||||
|
||||
|
||||
def _tensor_to_f32(weight) -> "object":
|
||||
"""任意 torch/numpy/list -> numpy float32 2D (in,out) 数组。"""
|
||||
import numpy as np
|
||||
if hasattr(weight, "detach"): # torch.Tensor
|
||||
weight = weight.detach().cpu().numpy()
|
||||
return np.asarray(weight, dtype=np.float32)
|
||||
|
||||
|
||||
Q8_BLOCK = 32 # int8 量化块大小
|
||||
|
||||
|
||||
def _tensor_bytes(w, dtype: int) -> bytes:
|
||||
"""把一个 2D f32 数组编码成 dtype 字节流（行主序展平）。"""
|
||||
import numpy as np
|
||||
w = np.ascontiguousarray(w, dtype=np.float32).reshape(-1)
|
||||
n = w.size
|
||||
if dtype == 0: # f32
|
||||
return w.tobytes()
|
||||
if dtype == 1: # fp16
|
||||
return w.astype(np.float16).astype("<f2").tobytes()
|
||||
if dtype == 2: # q8: 每块 [fp16 scale][32×int8]
|
||||
pad = (-n) % Q8_BLOCK
|
||||
wp = np.pad(w, (0, pad), "constant") if pad else w
|
||||
nb = wp.size // Q8_BLOCK
|
||||
b = wp.reshape(nb, Q8_BLOCK)
|
||||
scale = np.maximum(np.abs(b).max(axis=1), 1e-12)
|
||||
q = np.clip(np.round(b / scale[:, None] * 127.0), -127, 127).astype(np.int8)
|
||||
out = bytearray()
|
||||
for i in range(nb):
|
||||
out += np.float32(scale[i]).astype("<f2").tobytes() # 2 字节 fp16 scale
|
||||
out += q[i].tobytes()
|
||||
return bytes(out)
|
||||
raise ValueError(f"未知 dtype {dtype}")
|
||||
|
||||
|
||||
def _tensor_stored_bytes(rows, cols, dtype: int) -> int:
|
||||
n = rows * cols
|
||||
if dtype == 0:
|
||||
return n * 4
|
||||
if dtype == 1:
|
||||
return n * 2
|
||||
if dtype == 2:
|
||||
nb = (n + Q8_BLOCK - 1) // Q8_BLOCK
|
||||
return n + nb * 2
|
||||
raise ValueError(f"未知 dtype {dtype}")
|
||||
|
||||
|
||||
def save_pap(path: str, cfg: ModelConfig, vocab: List[bytes],
|
||||
merges: List[Tuple[int, int]], tensors: Dict[str, "object"], dtype: int = 0) -> None:
|
||||
"""写出 .pap 文件。tensors 的每个矩阵须为 (in, out) 行主序（C 端直接 x@W）。
|
||||
dtype: 0=f32, 1=fp16, 2=int8(块量化)。"""
|
||||
import numpy as np
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(PARZ_MAGIC)
|
||||
f.write(_u32(VERSION))
|
||||
f.write(_u32(dtype)) # dtype
|
||||
f.write(_u32(cfg.vocab_size))
|
||||
f.write(_u32(cfg.n_layer))
|
||||
f.write(_u32(cfg.d_model))
|
||||
f.write(_u32(cfg.n_head))
|
||||
f.write(_u32(cfg.d_ff))
|
||||
f.write(_u32(cfg.moe_n_experts))
|
||||
f.write(_u32(cfg.moe_top_k))
|
||||
f.write(_u32(cfg.d_expert))
|
||||
f.write(_u32(cfg.max_seq_len))
|
||||
f.write(_u32(cfg.num_merges))
|
||||
f.write(_u32(cfg.n_special))
|
||||
f.write(bytes([cfg.tie_weights & 0xFF, 0, 0, 0]))
|
||||
f.write(_f32(cfg.rmsnorm_eps))
|
||||
# moe_mask
|
||||
mask = bytes(cfg.moe_mask)
|
||||
assert len(mask) == cfg.n_layer
|
||||
f.write(mask)
|
||||
|
||||
# --- vocab blob ---
|
||||
blob = bytearray()
|
||||
for tok in vocab:
|
||||
blob += _u32(len(tok))
|
||||
blob += tok
|
||||
for (a, b) in merges:
|
||||
blob += _u32(a)
|
||||
blob += _u32(b)
|
||||
f.write(_u32(len(blob)))
|
||||
f.write(bytes(blob))
|
||||
|
||||
# --- weights ---
|
||||
names = sorted(tensors.keys())
|
||||
f.write(_u32(len(names)))
|
||||
for name in names:
|
||||
w = _tensor_to_f32(tensors[name])
|
||||
if w.ndim == 1:
|
||||
w = w.reshape(1, -1) # 一维 norm 向量按单行保存
|
||||
rows, cols = w.shape
|
||||
nb = name.encode("utf-8")
|
||||
f.write(_u32(len(nb)))
|
||||
f.write(nb)
|
||||
f.write(_u32(rows))
|
||||
f.write(_u32(cols))
|
||||
f.write(_tensor_bytes(w, dtype))
|
||||
|
||||
|
||||
def read_pap(path: str):
|
||||
"""读回 .pap，返回 (cfg, vocab, merges, tensors)。用于校验或转回 torch。"""
|
||||
import numpy as np
|
||||
|
||||
tensors: Dict[str, np.ndarray] = {}
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
off = 0
|
||||
off_r = lambda n: None
|
||||
|
||||
def r_u32(off):
|
||||
return (struct.unpack_from(_M + "I", data, off)[0], off + 4)
|
||||
|
||||
r_u8 = None
|
||||
|
||||
magic = data[0:4]
|
||||
assert magic == PARZ_MAGIC, f"bad magic {magic}"
|
||||
cfg = ModelConfig()
|
||||
|
||||
def u32(off):
|
||||
v, n = struct.unpack_from(_M + "I", data, off)[0], off + 4
|
||||
return v, n
|
||||
|
||||
def f32(off):
|
||||
v, n = struct.unpack_from(_M + "f", data, off)[0], off + 4
|
||||
return v, n
|
||||
|
||||
v, off = u32(4)
|
||||
assert v == VERSION
|
||||
dtype, off = u32(8)
|
||||
cfg.vocab_size, off = u32(12)
|
||||
cfg.n_layer, off = u32(16)
|
||||
cfg.d_model, off = u32(20)
|
||||
cfg.n_head, off = u32(24)
|
||||
cfg.d_ff, off = u32(28)
|
||||
cfg.moe_n_experts, off = u32(32)
|
||||
cfg.moe_top_k, off = u32(36)
|
||||
cfg.d_expert, off = u32(40)
|
||||
cfg.max_seq_len, off = u32(44)
|
||||
cfg.num_merges, off = u32(48)
|
||||
cfg.n_special, off = u32(52)
|
||||
cfg.tie_weights = data[56]
|
||||
off = 60
|
||||
cfg.rmsnorm_eps, off = f32(60)
|
||||
off = 64
|
||||
cfg.moe_mask = list(data[off:off + cfg.n_layer])
|
||||
off += cfg.n_layer
|
||||
|
||||
blob_len, off = u32(off)
|
||||
blob = data[off:off + blob_len]
|
||||
off += blob_len
|
||||
|
||||
def b32(o): # 在 blob 内读取
|
||||
return struct.unpack_from(_M + "I", blob, o)[0], o + 4
|
||||
|
||||
boff = 0
|
||||
vocab = []
|
||||
for _ in range(cfg.vocab_size):
|
||||
tl, boff = b32(boff)
|
||||
vocab.append(blob[boff:boff + tl])
|
||||
boff += tl
|
||||
merges = []
|
||||
for _ in range(cfg.num_merges):
|
||||
a, boff = b32(boff)
|
||||
b, boff = b32(boff)
|
||||
merges.append((a, b))
|
||||
|
||||
n_t, off = u32(off)
|
||||
for _ in range(n_t):
|
||||
nl, off = u32(off)
|
||||
name = data[off:off + nl].decode("utf-8")
|
||||
off += nl
|
||||
rows, off = u32(off)
|
||||
cols, off = u32(off)
|
||||
n = rows * cols
|
||||
if dtype == 0:
|
||||
arr = np.frombuffer(data, dtype=np.float32, count=n, offset=off).reshape(rows, cols).copy()
|
||||
off += n * 4
|
||||
elif dtype == 1:
|
||||
arr = np.frombuffer(data, dtype=np.float16, count=n, offset=off).astype(np.float32).reshape(rows, cols).copy()
|
||||
off += n * 2
|
||||
elif dtype == 2:
|
||||
nb = (n + Q8_BLOCK - 1) // Q8_BLOCK
|
||||
vals = np.empty(n, dtype=np.float32)
|
||||
po = off
|
||||
k = 0
|
||||
for _ in range(nb):
|
||||
scale = np.frombuffer(data, dtype=np.float16, count=1, offset=po).astype(np.float32)[0]
|
||||
po += 2
|
||||
cnt = min(Q8_BLOCK, n - k)
|
||||
q = np.frombuffer(data, dtype=np.int8, count=cnt, offset=po)
|
||||
po += cnt
|
||||
vals[k:k + cnt] = q.astype(np.float32) * (scale / 127.0)
|
||||
k += cnt
|
||||
arr = vals.reshape(rows, cols).copy()
|
||||
off += nb * (2 + Q8_BLOCK) # 每块 2 字节 scale + 32 字节 q
|
||||
else:
|
||||
raise ValueError(f"未知 dtype {dtype}")
|
||||
tensors[name] = arr
|
||||
return cfg, vocab, merges, tensors
|
||||
|
||||
|
||||
def pap_quant(src: str, dst: str, dtype: int) -> None:
|
||||
"""把已有 .pap 转成另一种 dtype（不改变词表/结构）。dtype: 1=fp16, 2=q8。"""
|
||||
cfg, vocab, merges, tensors = read_pap(src)
|
||||
save_pap(dst, cfg, vocab, merges, tensors, dtype=dtype)
|
||||
@@ -0,0 +1,38 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""把 .pap 模型转成更小的 dtype（能力基本不变）。
|
||||
|
||||
用法:
|
||||
/opt/pap-venv/bin/python platform/quant_pap.py \
|
||||
--src models/moe-0.1b.pap --dst models/moe-0.1b-fp16.pap --dtype fp16
|
||||
|
||||
--dtype fp16: f32 -> 半精度, 体积约减半 (437MB -> ~219MB), 能力几乎不变
|
||||
--dtype q8 : f32 -> int8 块量化(每32个共享fp16 scale), 约 437/4 (~110MB), 损失略大
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
import pap_format
|
||||
|
||||
DTYPES = {"f32": 0, "fp16": 1, "q8": 2}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--src", required=True)
|
||||
ap.add_argument("--dst", required=True)
|
||||
ap.add_argument("--dtype", default="fp16", choices=DTYPES.keys())
|
||||
args = ap.parse_args()
|
||||
|
||||
dst_dtype = DTYPES[args.dtype]
|
||||
pap_format.pap_quant(args.src, args.dst, dst_dtype)
|
||||
|
||||
in_sz = os.path.getsize(args.src)
|
||||
out_sz = os.path.getsize(args.dst)
|
||||
print(f"已转换: {args.src} ({in_sz/1e6:.1f} MB) -> {args.dst} ({out_sz/1e6:.1f} MB)")
|
||||
print(f" 压缩比 {in_sz/out_sz:.2f}x dtype={args.dtype} (code {dst_dtype})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
ParlzAIPlatformPAP — byte-level BPE tokenizer.
|
||||
与 C 端 (src/model/tokenizer.c) 完全一致的算法:
|
||||
- id 0..255 = 单个字节 bytes([b])
|
||||
- id 256+ = BPE 合并产生的 token (vocab[base+i])
|
||||
- merges[i] = (a, b), 合并后得到 id = 256 + i
|
||||
编码采用“全局最小 rank 优先”的 BPE（与 C 端一致），用堆实现 O(n log n)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import heapq
|
||||
from typing import List, Tuple
|
||||
|
||||
|
||||
def train_bpe(text: str, vocab_size: int) -> Tuple[List[bytes], List[Tuple[int, int]]]:
|
||||
"""在字节序列上训练 BPE，返回 (vocab, merges)。"""
|
||||
raw = text.encode("utf-8")
|
||||
ids: List[int] = list(raw)
|
||||
|
||||
vocab: List[bytes] = [bytes([b]) for b in range(256)]
|
||||
|
||||
def get_stats(ids: List[int]) -> dict:
|
||||
st: dict = {}
|
||||
for i in range(len(ids) - 1):
|
||||
pair = (ids[i], ids[i + 1])
|
||||
st[pair] = st.get(pair, 0) + 1
|
||||
return st
|
||||
|
||||
merges: List[Tuple[int, int]] = []
|
||||
while len(vocab) < vocab_size:
|
||||
stats = get_stats(ids)
|
||||
if not stats:
|
||||
break
|
||||
best = max(stats.items(), key=lambda kv: (kv[1], -kv[0][0], -kv[0][1]))[0]
|
||||
(a, b) = best
|
||||
new_id = len(vocab)
|
||||
vocab.append(vocab[a] + vocab[b])
|
||||
merges.append((a, b))
|
||||
new_ids: List[int] = []
|
||||
i = 0
|
||||
while i < len(ids):
|
||||
if i < len(ids) - 1 and ids[i] == a and ids[i + 1] == b:
|
||||
new_ids.append(new_id)
|
||||
i += 2
|
||||
else:
|
||||
new_ids.append(ids[i])
|
||||
i += 1
|
||||
ids = new_ids
|
||||
return vocab, merges
|
||||
|
||||
|
||||
def build_ranks(merges: List[Tuple[int, int]]) -> dict:
|
||||
return {pair: i for i, pair in enumerate(merges)}
|
||||
|
||||
|
||||
def encode(text: str, vocab_size: int, merges: List[Tuple[int, int]],
|
||||
bos: bool = False, eos: bool = False, bos_id: int = 0, eos_id: int = 0) -> List[int]:
|
||||
"""全局最小 rank 的 BPE 编码；结果与 C 端 tokenizer_encode 一致。"""
|
||||
ranks = {pair: i for i, pair in enumerate(merges)}
|
||||
if not ranks:
|
||||
ids = list(text.encode("utf-8"))
|
||||
else:
|
||||
ids = _bpe_merge(list(text.encode("utf-8")), ranks)
|
||||
if bos:
|
||||
ids = [bos_id] + ids if bos_id != 0 else ids
|
||||
if eos:
|
||||
ids = ids + [eos_id]
|
||||
return ids
|
||||
|
||||
|
||||
def _bpe_merge(ids: List[int], ranks: dict) -> List[int]:
|
||||
n = len(ids)
|
||||
if n < 2:
|
||||
return ids
|
||||
base = 256
|
||||
prev = [i - 1 for i in range(n)]
|
||||
nxt = [i + 1 for i in range(n)]
|
||||
nxt[n - 1] = -1
|
||||
alive = [True] * n
|
||||
|
||||
heap: List[Tuple[int, int]] = []
|
||||
for i in range(n - 1):
|
||||
r = ranks.get((ids[i], ids[i + 1]))
|
||||
if r is not None:
|
||||
heapq.heappush(heap, (r, i))
|
||||
|
||||
while heap:
|
||||
rank, i = heapq.heappop(heap)
|
||||
if not alive[i]:
|
||||
continue
|
||||
j = nxt[i]
|
||||
if j < 0:
|
||||
continue
|
||||
cur = ranks.get((ids[i], ids[j]))
|
||||
if cur != rank: # 过期条目：隔壁合并改变了该 pair
|
||||
if cur is not None:
|
||||
heapq.heappush(heap, (cur, i))
|
||||
continue
|
||||
# 合并 i,j -> i 保留，j 标记死亡
|
||||
ids[i] = base + rank
|
||||
alive[j] = False
|
||||
k = nxt[j]
|
||||
nxt[i] = k
|
||||
if k >= 0:
|
||||
prev[k] = i
|
||||
# 受影响的两个关节： (prev[i], i) 与 (i, k)
|
||||
pj = prev[i]
|
||||
if pj >= 0 and alive[pj]:
|
||||
r = ranks.get((ids[pj], ids[i]))
|
||||
if r is not None:
|
||||
heapq.heappush(heap, (r, pj))
|
||||
if k >= 0:
|
||||
r = ranks.get((ids[i], ids[k]))
|
||||
if r is not None:
|
||||
heapq.heappush(heap, (r, i))
|
||||
|
||||
return [ids[i] for i in range(n) if alive[i]]
|
||||
|
||||
|
||||
def decode(ids: List[int], vocab: List[bytes]) -> str:
|
||||
buf = b"".join(vocab[i] for i in ids if 0 <= i < len(vocab))
|
||||
return buf.decode("utf-8", errors="replace")
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
ParlzAIPlatformPAP — 训练器。
|
||||
|
||||
从 (可选的) 语料文件训练一个 GPT+MoE 模型，并导出为 ParlzMAI 可运行的 .pap 文件。
|
||||
|
||||
用法（在 WSL 内，使用含 torch 的 venv）:
|
||||
/opt/pap-venv/bin/python platform/train.py --data data/tinyshakespeare.txt \
|
||||
--out models/moe-0.1b.pap --vocab 2048 --layers 6 --d_model 512 --heads 8 \
|
||||
--experts 16 --top_k 2 --d_expert 1024 --max_seq 256 --steps 1500 --batch 8
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
import tokenizer as tok
|
||||
import pap_format
|
||||
from model import ParlzGPTMoE, export_tensors
|
||||
|
||||
|
||||
def build_config(vocab_size, n_layer, d_model, n_head, d_ff, moe_n_experts,
|
||||
moe_top_k, d_expert, max_seq, eps, moe_every):
|
||||
cfg = {
|
||||
"vocab_size": vocab_size,
|
||||
"n_layer": n_layer,
|
||||
"d_model": d_model,
|
||||
"n_head": n_head,
|
||||
"d_ff": d_ff,
|
||||
"moe_n_experts": moe_n_experts,
|
||||
"moe_top_k": moe_top_k,
|
||||
"d_expert": d_expert,
|
||||
"max_seq_len": max_seq,
|
||||
"rmsnorm_eps": eps,
|
||||
# 每层是否 MoE；moe_every=1 表示全部 MoE，=2 表示隔层
|
||||
"moe_mask": [1 if (l % moe_every == 0) else 0 for l in range(n_layer)],
|
||||
}
|
||||
return cfg
|
||||
|
||||
|
||||
def count_params(model):
|
||||
return sum(p.numel() for p in model.parameters())
|
||||
|
||||
|
||||
def get_batch(data_t, start, block_size, batch_size, device):
|
||||
"""取一个 batch: x = [batch, block_size], y 右移一位。"""
|
||||
ix = torch.randint(0, data_t.numel() - block_size, (batch_size,))
|
||||
x = torch.stack([data_t[i:i + block_size] for i in ix])
|
||||
y = torch.stack([data_t[i + 1:i + 1 + block_size] for i in ix])
|
||||
return x.to(device), y.to(device)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--data", default="data/tinyshakespeare.txt")
|
||||
ap.add_argument("--out", default="models/moe-0.1b.pap")
|
||||
ap.add_argument("--vocab", type=int, default=2048)
|
||||
ap.add_argument("--layers", type=int, default=6)
|
||||
ap.add_argument("--d_model", type=int, default=512)
|
||||
ap.add_argument("--heads", type=int, default=8)
|
||||
ap.add_argument("--d_ff", type=int, default=2048)
|
||||
ap.add_argument("--experts", type=int, default=16)
|
||||
ap.add_argument("--top_k", type=int, default=2)
|
||||
ap.add_argument("--d_expert", type=int, default=1024)
|
||||
ap.add_argument("--max_seq", type=int, default=256)
|
||||
ap.add_argument("--moe_every", type=int, default=1)
|
||||
ap.add_argument("--eps", type=float, default=1e-5)
|
||||
ap.add_argument("--steps", type=int, default=1500)
|
||||
ap.add_argument("--batch", type=int, default=8)
|
||||
ap.add_argument("--lr", type=float, default=3e-4)
|
||||
ap.add_argument("--warmup", type=int, default=100)
|
||||
ap.add_argument("--eval_every", type=int, default=200)
|
||||
ap.add_argument("--seed", type=int, default=1337)
|
||||
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
args = ap.parse_args()
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
device = args.device
|
||||
if device.startswith("cuda") and not torch.cuda.is_available():
|
||||
print("CUDA 不可用，回退到 CPU", flush=True)
|
||||
device = "cpu"
|
||||
|
||||
# ---- 读语料 + BPE（带缓存） ----
|
||||
if not os.path.exists(args.data):
|
||||
raise SystemExit(f"语料不存在: {args.data}")
|
||||
text = open(args.data, "r", encoding="utf-8").read()
|
||||
print(f"语料: {len(text)} 字符", flush=True)
|
||||
|
||||
cache = args.data + ".papcache"
|
||||
if os.path.exists(cache):
|
||||
import pickle as _pk
|
||||
with open(cache, "rb") as f:
|
||||
vocab, merges, data_ids = _pk.load(f)
|
||||
vocab_size = len(vocab)
|
||||
print(f"加载预处理缓存: vocab={vocab_size} merges={len(merges)}", flush=True)
|
||||
else:
|
||||
import pickle as _pk
|
||||
vocab, merges = tok.train_bpe(text, args.vocab)
|
||||
vocab_size = len(vocab)
|
||||
data_ids = tok.encode(text, vocab_size, merges)
|
||||
with open(cache, "wb") as f:
|
||||
_pk.dump((vocab, merges, data_ids), f)
|
||||
print(f"词表: {vocab_size} (字节+合并)", flush=True)
|
||||
|
||||
# ---- 构建模型 ----
|
||||
cfg = build_config(vocab_size, args.layers, args.d_model, args.heads, args.d_ff,
|
||||
args.experts, args.top_k, args.d_expert, args.max_seq,
|
||||
args.eps, args.moe_every)
|
||||
model = ParlzGPTMoE(cfg).to(device)
|
||||
nparams = count_params(model)
|
||||
print(f"模型参数: {nparams/1e6:.1f} M", flush=True)
|
||||
|
||||
# ---- 数据 ----
|
||||
data_ids = tok.encode(text, vocab_size, merges)
|
||||
data_t = torch.tensor(data_ids, dtype=torch.long)
|
||||
block = args.max_seq
|
||||
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.1,
|
||||
betas=(0.9, 0.95))
|
||||
sched = torch.optim.lr_scheduler.LambdaLR(
|
||||
optimizer,
|
||||
lambda step: min((step + 1) / max(args.warmup, 1), 1.0))
|
||||
model.train()
|
||||
|
||||
print(f"开始训练 {args.steps} 步 (device={device}) ...", flush=True)
|
||||
t0 = time.time()
|
||||
for step in range(1, args.steps + 1):
|
||||
x, y = get_batch(data_t, 0, block, args.batch, device)
|
||||
loss, logits = (None, None)
|
||||
out = model(x, y)
|
||||
loss = out[1]
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
||||
optimizer.step()
|
||||
sched.step()
|
||||
|
||||
if step % args.eval_every == 0 or step == 1:
|
||||
el = time.time() - t0
|
||||
lr = optimizer.param_groups[0]["lr"]
|
||||
print(f" step {step:5d}|\tloss {loss.item():.4f}\tlr {lr:.2e}\t{el:.1f}s", flush=True)
|
||||
model.eval()
|
||||
init_ids = tok.encode("The", vocab_size, merges)
|
||||
gen_ids = model.generate_from_ids(init_ids, 60, temperature=0.8, top_k=40)
|
||||
gen = tok.decode(gen_ids, vocab)
|
||||
if gen:
|
||||
print(f" [sample] {gen}", flush=True)
|
||||
model.train()
|
||||
|
||||
print(f"训练完成，耗时 {time.time()-t0:.1f}s", flush=True)
|
||||
model.eval()
|
||||
|
||||
# ---- 导出 .pap ----
|
||||
tens = export_tensors(model)
|
||||
pcfg = pap_format.ModelConfig()
|
||||
pcfg.vocab_size = vocab_size
|
||||
pcfg.n_layer = args.layers
|
||||
pcfg.d_model = args.d_model
|
||||
pcfg.n_head = args.heads
|
||||
pcfg.d_ff = args.d_ff
|
||||
pcfg.moe_n_experts = args.experts
|
||||
pcfg.moe_top_k = args.top_k
|
||||
pcfg.d_expert = args.d_expert
|
||||
pcfg.max_seq_len = args.max_seq
|
||||
pcfg.num_merges = len(merges)
|
||||
pcfg.tie_weights = 0
|
||||
pcfg.rmsnorm_eps = args.eps
|
||||
pcfg.moe_mask = [int(v) for v in cfg["moe_mask"]]
|
||||
|
||||
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
||||
pap_format.save_pap(args.out, pcfg, vocab, merges, tens)
|
||||
print(f"已导出 .pap -> {args.out} ({os.path.getsize(args.out)/1e6:.1f} MB)", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""对拍：torch 贪心 vs C 贪心，验证两端数学一致。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
|
||||
import model as model_mod
|
||||
import tokenizer as tok
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--pap", required=True)
|
||||
ap.add_argument("--prompt", default="First Citizen:")
|
||||
ap.add_argument("--n", type=int, default=40)
|
||||
ap.add_argument("--cbin", default="/mnt/f/ParlzMAI/build/moe-serve")
|
||||
args = ap.parse_args()
|
||||
|
||||
m, vocab, merges = model_mod.load_pap_model(args.pap)
|
||||
init = tok.encode(args.prompt, m.cfg["vocab_size"], merges)
|
||||
ids = m.generate_from_ids(init, args.n, temperature=0) # greedy
|
||||
py_text = tok.decode(ids, vocab)
|
||||
|
||||
out = subprocess.run(
|
||||
[args.cbin, "--model", args.pap, "--prompt", args.prompt,
|
||||
"--n_tokens", str(args.n), "--temperature", "0", "--top_k", "0"],
|
||||
capture_output=True)
|
||||
c_all = out.stdout.decode("utf-8", errors="replace")
|
||||
idx = c_all.rfind(args.prompt)
|
||||
c_text = c_all[idx:] if idx >= 0 else c_all
|
||||
|
||||
print("=== torch greedy ===")
|
||||
print(py_text)
|
||||
print("=== C greedy ===")
|
||||
print(c_text.strip())
|
||||
ok = c_text.strip() == py_text.strip()
|
||||
print(f"\n[MATCH] {'YES' if ok else 'NO'}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,58 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""对拍：llama 架构 C(torch 数学) — 合成随机 MoE 模型 → GGUF → C 加载运行 → 与 torch 贪心 ids 一致。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
|
||||
import llama_model as lm
|
||||
import gguf_format
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="/mnt/f/ParlzMAI/models/_llama_smoke.gguf")
|
||||
ap.add_argument("--cbin", default="/mnt/f/ParlzMAI/build/moe-llama")
|
||||
ap.add_argument("--n", type=int, default=8)
|
||||
ap.add_argument("--init", default="1 42 100")
|
||||
ap.add_argument("--device", default="cpu")
|
||||
args = ap.parse_args()
|
||||
|
||||
torch = __import__("torch")
|
||||
torch.manual_seed(1234)
|
||||
|
||||
cfg = lm.LlamaConfig(vocab_size=512, n_layer=2, n_embd=128, n_head=8, n_head_kv=4,
|
||||
ffn_dim=256, n_expert=4, n_expert_used=2, max_seq=24,
|
||||
rmsnorm_eps=1e-5, rope_theta=10000.0)
|
||||
|
||||
model = lm.LlamaMoE(cfg)
|
||||
tens = lm.export_tensors(model)
|
||||
# 字节 tokenizer（合成）: tokens = chr(b), merges = []
|
||||
tokens = [chr(b) for b in range(256)]
|
||||
merges = []
|
||||
gguf_format.save_llama_gguf(args.out, cfg, tens, gml_type=1, tokens=tokens, merges=merges)
|
||||
|
||||
# torch 端：从 GGUF 重建（严格圆回）
|
||||
m2 = lm.load_llama_from_gguf(args.out, cfg)
|
||||
m2.to("cpu")
|
||||
init = [int(x) for x in args.init.split()]
|
||||
py_ids = m2.generate_from_ids(init, args.n, temperature=0) # 贪心
|
||||
py_gen = py_ids[len(init):] # 去掉 init 前缀
|
||||
|
||||
# C 端
|
||||
cmd = [args.cbin, args.out, str(args.n)] + [str(x) for x in init]
|
||||
out = subprocess.run(cmd, capture_output=True, text=True)
|
||||
c_ids = [int(x) for x in out.stdout.split()]
|
||||
|
||||
print("=== torch 贪心 ids ===")
|
||||
print(py_ids)
|
||||
print("=== C 贪心 ids ===")
|
||||
print(c_ids)
|
||||
ok = py_gen == c_ids
|
||||
print(f"\n[MATCH] {'YES' if ok else 'NO'}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(main())
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# 基准：测 pmai 生成速度（tokens/s 与端到端延迟）
|
||||
# 用法: scripts/bench.sh <model> [n_tokens]
|
||||
set -euo pipefail
|
||||
MODEL="${1:-../models/moe-0.1b-fp16.pap}"
|
||||
N="${2:-100}"
|
||||
CDIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
BIN="$CDIR/output/pmai"
|
||||
|
||||
echo "== pmai 基准: $MODEL n=$N =="
|
||||
start=$(date +%s.%N)
|
||||
"$BIN" "$MODEL" --prompt "The" --n_tokens "$N" --temperature 0 --top_k 0 > /tmp/bench_out.txt
|
||||
end=$(date +%s.%N)
|
||||
|
||||
dt=$(echo "$end - $start" | bc)
|
||||
tok=$(echo "$N" | bc)
|
||||
tps=$(echo "scale=2; $tok / $dt" | bc)
|
||||
echo "耗时: ${dt}s 约 ${tps} tokens/s（含模型加载）"
|
||||
|
||||
# 仅生成时间（预热一次）
|
||||
start=$(date +%s.%N)
|
||||
"$BIN" "$MODEL" --prompt "The" --n_tokens "$N" --temperature 0 --top_k 0 > /dev/null
|
||||
end=$(date +%s.%N)
|
||||
echo "第二次（已缓存）: $(echo "$end - $start" | bc)s ~$(echo "scale=2; $N / ($end - $start)" | bc) tokens/s"
|
||||
@@ -0,0 +1,39 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""从 HF 镜像下载 GGUF / AI 模型文件。
|
||||
用法: /opt/pap-venv/bin/python scripts/download_model.py --repo Qwen/Qwen2.5-0.5B-Instruct-GGUF \
|
||||
--file qwen2.5-0.5b-instruct-q4_k_m.gguf --out models/
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--repo", required=True, help="HF 仓库（如 ggml-org/stories15M_MOE）")
|
||||
ap.add_argument("--file", required=True, help="仓库内文件名")
|
||||
ap.add_argument("--out", default="models/", help="输出目录")
|
||||
ap.add_argument("--base", default="https://hf-mirror.com", help="镜像")
|
||||
args = ap.parse_args()
|
||||
|
||||
url = f"{args.base}/{args.repo}/resolve/main/{args.file}"
|
||||
os.makedirs(args.out, exist_ok=True)
|
||||
dst = os.path.join(args.out, os.path.basename(args.file))
|
||||
print(f"下载 {url} -> {dst}")
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=600) as r, open(dst, "wb") as f:
|
||||
total = 0
|
||||
while True:
|
||||
chunk = r.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
total += len(chunk)
|
||||
print(f"\r {total/1e6:.1f} MB", end="")
|
||||
print(f"\n完成: {dst} ({total/1e6:.1f} MB)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# 等待 pap-venv 的 torch/numpy 装好（用于 WSL 内训练）
|
||||
for i in $(seq 1 40); do
|
||||
if /opt/pap-venv/bin/python -c 'import torch,numpy' >/dev/null 2>&1; then
|
||||
echo "READY"
|
||||
/opt/pap-venv/bin/python -c 'import torch,numpy;print("torch",torch.__version__,"cuda",torch.cuda.is_available());print("numpy",numpy.__version__)' 2>&1 | head -3
|
||||
exit 0
|
||||
fi
|
||||
sleep 6
|
||||
done
|
||||
echo "NOT_READY_AFTER_240s"
|
||||
exit 1
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# 等训练日志出现关键行：语料/词表/参数/步数
|
||||
for i in $(seq 1 60); do
|
||||
if grep -qE "step +1|已导出|Traceback|Error|CUDA" /tmp/pap_train.log 2>/dev/null; then
|
||||
echo "FOUND_AT_${i}"; break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "----- tail -----"
|
||||
tail -30 /tmp/pap_train.log 2>/dev/null | tr -d '\0'
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
#include "core/matrix.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* 简单可复现的 xorshift RNG（避免 srand 的跨平台差异） */
|
||||
static uint32_t g_rng = 2463534242u;
|
||||
|
||||
static uint32_t next_rng(void) {
|
||||
uint32_t x = g_rng;
|
||||
x ^= x << 13;
|
||||
x ^= x >> 17;
|
||||
x ^= x << 5;
|
||||
g_rng = x;
|
||||
return x;
|
||||
}
|
||||
|
||||
static float uniform01(void) {
|
||||
return (float)(next_rng() & 0xFFFFFFu) / (float)(1u << 24);
|
||||
}
|
||||
|
||||
MoMat *mat_alloc(int rows, int cols) {
|
||||
if (rows < 0 || cols < 0) return NULL;
|
||||
MoMat *m = calloc(1, sizeof(MoMat));
|
||||
if (!m) return NULL;
|
||||
m->data = calloc((size_t)rows * (size_t)cols, sizeof(float));
|
||||
if (!m->data) {
|
||||
free(m);
|
||||
return NULL;
|
||||
}
|
||||
m->rows = rows;
|
||||
m->cols = cols;
|
||||
m->stride = cols;
|
||||
m->owns = 1;
|
||||
return m;
|
||||
}
|
||||
|
||||
void mat_free(MoMat *m) {
|
||||
if (!m) return;
|
||||
if (m->owns && m->data) free(m->data);
|
||||
free(m);
|
||||
}
|
||||
|
||||
MoMat *mat_view(MoMat *m, int rows, int cols, int64_t stride, float *data) {
|
||||
if (!m) return NULL;
|
||||
m->rows = rows;
|
||||
m->cols = cols;
|
||||
m->stride = stride;
|
||||
m->data = data;
|
||||
m->owns = 0;
|
||||
return m;
|
||||
}
|
||||
|
||||
void mat_fill(MoMat *m, float value) {
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
for (int j = 0; j < m->cols; j++) row[j] = value;
|
||||
}
|
||||
}
|
||||
|
||||
void mat_randn(MoMat *m, unsigned seed, float stddev) {
|
||||
if (seed) g_rng = seed;
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
for (int j = 0; j < m->cols; j++) {
|
||||
/* Box-Muller：两个均匀 -> 两个高斯，用其中一个 */
|
||||
float u1 = uniform01();
|
||||
if (u1 <= 0.0f) u1 = 1e-9f;
|
||||
float u2 = uniform01();
|
||||
float z = sqrtf(-2.0f * logf(u1)) * cosf(2.0f * 3.14159265358979323846f * u2);
|
||||
row[j] = z * stddev;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mat_mul(const MoMat *a, const MoMat *b, MoMat *out) {
|
||||
if (!a || !b || !out) return -1;
|
||||
int M = a->rows;
|
||||
int K = a->cols;
|
||||
int N = b->cols;
|
||||
if (b->rows != K) return -1;
|
||||
if (out->rows != M || out->cols != N) return -1;
|
||||
|
||||
for (int i = 0; i < M; i++) {
|
||||
const float *pa = a->data + (int64_t)i * a->stride;
|
||||
float *po = out->data + (int64_t)i * out->stride;
|
||||
for (int j = 0; j < N; j++) {
|
||||
float s = 0.0f;
|
||||
for (int k = 0; k < K; k++) {
|
||||
s += pa[k] * b->data[(int64_t)k * b->stride + j];
|
||||
}
|
||||
po[j] = s;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void mat_add(MoMat *a, const MoMat *b) {
|
||||
if (!a || !b || a->rows != b->rows || a->cols != b->cols) return;
|
||||
for (int i = 0; i < a->rows; i++) {
|
||||
float *pa = a->data + (int64_t)i * a->stride;
|
||||
const float *pb = b->data + (int64_t)i * b->stride;
|
||||
for (int j = 0; j < a->cols; j++) pa[j] += pb[j];
|
||||
}
|
||||
}
|
||||
|
||||
void mat_scale(MoMat *m, float factor) {
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
for (int j = 0; j < m->cols; j++) row[j] *= factor;
|
||||
}
|
||||
}
|
||||
|
||||
MoMat *mat_transpose(const MoMat *a, MoMat *dst) {
|
||||
if (!a || !dst) return NULL;
|
||||
if (dst->rows != a->cols || dst->cols != a->rows) return NULL;
|
||||
for (int i = 0; i < a->rows; i++) {
|
||||
const float *pa = a->data + (int64_t)i * a->stride;
|
||||
for (int j = 0; j < a->cols; j++) {
|
||||
dst->data[(int64_t)j * dst->stride + i] = pa[j];
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
void mat_softmax_rows(MoMat *m) {
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
/* 数值稳定：减去行内最大值 */
|
||||
float mx = row[0];
|
||||
for (int j = 1; j < m->cols; j++) if (row[j] > mx) mx = row[j];
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int j = 0; j < m->cols; j++) {
|
||||
row[j] = expf(row[j] - mx);
|
||||
sum += row[j];
|
||||
}
|
||||
float inv = 1.0f / sum;
|
||||
for (int j = 0; j < m->cols; j++) row[j] *= inv;
|
||||
}
|
||||
}
|
||||
|
||||
void mat_relu(MoMat *m) {
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
for (int j = 0; j < m->cols; j++) if (row[j] < 0.0f) row[j] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
void mat_sigmoid(MoMat *m) {
|
||||
for (int i = 0; i < m->rows; i++) {
|
||||
float *row = m->data + (int64_t)i * m->stride;
|
||||
for (int j = 0; j < m->cols; j++) {
|
||||
float x = row[j];
|
||||
row[j] = x >= 0.0f ? (1.0f / (1.0f + expf(-x)))
|
||||
: (expf(x) / (1.0f + expf(x)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#ifndef MO_MATRIX_H
|
||||
#define MO_MATRIX_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* 浮点矩阵（行主序）。支持 stride，方便对张量切片做视图而不复制。
|
||||
* Phase 1 提供基础算子（串行）；后续用 OpenMP/BLAS 加速。
|
||||
*/
|
||||
typedef struct {
|
||||
int rows;
|
||||
int cols;
|
||||
int64_t stride; /* 相邻两行之间的浮点偏移 */
|
||||
float *data;
|
||||
uint8_t owns; /* 是否负责释放 data */
|
||||
} MoMat;
|
||||
|
||||
/* 分配 rows x cols 全 0 矩阵。失败返回 NULL */
|
||||
MoMat *mat_alloc(int rows, int cols);
|
||||
|
||||
/* 释放矩阵（data 仅当 owns 为真时释放） */
|
||||
void mat_free(MoMat *m);
|
||||
|
||||
/* 用外部缓冲构造一个视图（不复制、不释放） */
|
||||
MoMat *mat_view(MoMat *m, int rows, int cols, int64_t stride, float *data);
|
||||
|
||||
/* 填充常量 */
|
||||
void mat_fill(MoMat *m, float value);
|
||||
|
||||
/* 高斯随机初始化（Box-Muller；seed=0 恢复默认序列） */
|
||||
void mat_randn(MoMat *m, unsigned seed, float stddev);
|
||||
|
||||
/* out = a * b。维度须满足 (M,K)x(K,N)->(M,N)。成功返回 0 */
|
||||
int mat_mul(const MoMat *a, const MoMat *b, MoMat *out);
|
||||
|
||||
/* a += b（同尺寸） */
|
||||
void mat_add(MoMat *a, const MoMat *b);
|
||||
|
||||
/* m *= factor */
|
||||
void mat_scale(MoMat *m, float factor);
|
||||
|
||||
/* dst = transpose(a)。dst 尺寸应为 (cols, rows)；返回 dst */
|
||||
MoMat *mat_transpose(const MoMat *a, MoMat *dst);
|
||||
|
||||
/* 逐行 softmax（原地） */
|
||||
void mat_softmax_rows(MoMat *m);
|
||||
|
||||
/* ReLU 原地 */
|
||||
void mat_relu(MoMat *m);
|
||||
|
||||
/* Sigmoid 原地 */
|
||||
void mat_sigmoid(MoMat *m);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_MATRIX_H */
|
||||
@@ -0,0 +1,95 @@
|
||||
#include "core/tensor.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static int64_t prod_dims(int ndim, const int64_t *shape) {
|
||||
int64_t p = 1;
|
||||
for (int i = 0; i < ndim; i++) p *= shape[i];
|
||||
return p;
|
||||
}
|
||||
|
||||
static void compute_strides(int ndim, const int64_t *shape, int64_t *strides) {
|
||||
int64_t s = 1;
|
||||
for (int i = ndim - 1; i >= 0; i--) {
|
||||
strides[i] = s;
|
||||
s *= shape[i];
|
||||
}
|
||||
}
|
||||
|
||||
MoTensor *tensor_create(int ndim, const int64_t *shape) {
|
||||
if (ndim <= 0 || !shape) return NULL;
|
||||
|
||||
MoTensor *t = calloc(1, sizeof(MoTensor));
|
||||
if (!t) return NULL;
|
||||
|
||||
t->ndim = ndim;
|
||||
t->shape = malloc((size_t)ndim * sizeof(int64_t));
|
||||
t->strides = malloc((size_t)ndim * sizeof(int64_t));
|
||||
if (!t->shape || !t->strides) {
|
||||
free(t->shape);
|
||||
free(t->strides);
|
||||
free(t);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memcpy(t->shape, shape, (size_t)ndim * sizeof(int64_t));
|
||||
compute_strides(ndim, shape, t->strides);
|
||||
|
||||
t->numel = prod_dims(ndim, shape);
|
||||
t->data = calloc((size_t)t->numel, sizeof(float));
|
||||
t->owns = 1;
|
||||
if (!t->data) {
|
||||
free(t->shape);
|
||||
free(t->strides);
|
||||
free(t);
|
||||
return NULL;
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
MoTensor *tensor_create_zeros(int ndim, const int64_t *shape) {
|
||||
return tensor_create(ndim, shape); /* calloc 已将 data 清零 */
|
||||
}
|
||||
|
||||
void tensor_free(MoTensor *t) {
|
||||
if (!t) return;
|
||||
if (t->owns && t->data) free(t->data);
|
||||
free(t->shape);
|
||||
free(t->strides);
|
||||
free(t);
|
||||
}
|
||||
|
||||
MoTensor *tensor_clone(const MoTensor *t) {
|
||||
if (!t) return NULL;
|
||||
MoTensor *c = tensor_create(t->ndim, t->shape);
|
||||
if (!c) return NULL;
|
||||
memcpy(c->data, t->data, (size_t)t->numel * sizeof(float));
|
||||
return c;
|
||||
}
|
||||
|
||||
void tensor_fill(MoTensor *t, float value) {
|
||||
if (!t) return;
|
||||
for (int64_t i = 0; i < t->numel; i++) t->data[i] = value;
|
||||
}
|
||||
|
||||
int64_t tensor_numel(const MoTensor *t) { return t ? t->numel : 0; }
|
||||
int64_t tensor_size(const MoTensor *t, int dim) { return t ? t->shape[dim] : 0; }
|
||||
|
||||
int64_t tensor_offset(const MoTensor *t, const int64_t *indices) {
|
||||
int64_t off = 0;
|
||||
for (int i = 0; i < t->ndim; i++) {
|
||||
off += indices[i] * t->strides[i];
|
||||
}
|
||||
return off;
|
||||
}
|
||||
|
||||
MoTensor *tensor_reshape(MoTensor *t, int ndim, const int64_t *shape) {
|
||||
if (!t || ndim <= 0 || !shape) return NULL;
|
||||
if (prod_dims(ndim, shape) != t->numel) return NULL; /* 元素总数不一致 */
|
||||
|
||||
memcpy(t->shape, shape, (size_t)ndim * sizeof(int64_t));
|
||||
compute_strides(ndim, shape, t->strides);
|
||||
t->ndim = ndim;
|
||||
return t; /* 就地改写，返回同指针 */
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
#ifndef MO_TENSOR_H
|
||||
#define MO_TENSOR_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* 多维张量。行主序（row-major）连续存储，浮点单精度。
|
||||
* 参考 llama2.c / Qwen 的分层思路，Phase 1 先做基础结构与访问。
|
||||
*/
|
||||
typedef struct {
|
||||
int ndim; /* 维度数量 */
|
||||
int64_t *shape; /* 每个维度大小，长度 ndim */
|
||||
int64_t *strides; /* 每个维度的元素步长（行主序），长度 ndim */
|
||||
int64_t numel; /* 元素总数 = prod(shape) */
|
||||
float *data; /* 连续元素缓冲（行主序） */
|
||||
uint8_t owns; /* data 是否为本对象所有（由本对象负责释放） */
|
||||
} MoTensor;
|
||||
|
||||
/* 创建全 0 张量（赋值阻塞维度语义）。失败返回 NULL */
|
||||
MoTensor *tensor_create(int ndim, const int64_t *shape);
|
||||
MoTensor *tensor_create_zeros(int ndim, const int64_t *shape);
|
||||
|
||||
/* 释放张量（data 仅当 owns 为真时释放） */
|
||||
void tensor_free(MoTensor *t);
|
||||
|
||||
/* 深拷贝（新的 data 缓冲区） */
|
||||
MoTensor *tensor_clone(const MoTensor *t);
|
||||
|
||||
/* 用标量填充 */
|
||||
void tensor_fill(MoTensor *t, float value);
|
||||
|
||||
/* 元素总数 */
|
||||
int64_t tensor_numel(const MoTensor *t);
|
||||
/* 某维大小 */
|
||||
int64_t tensor_size(const MoTensor *t, int dim);
|
||||
|
||||
/* 计算给定下标（长度 ndim）的线性偏移 */
|
||||
int64_t tensor_offset(const MoTensor *t, const int64_t *indices);
|
||||
|
||||
/* 就地 reshape 视图（不复制数据）。元素总数必须一致，否则返回 NULL */
|
||||
MoTensor *tensor_reshape(MoTensor *t, int ndim, const int64_t *shape);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_TENSOR_H */
|
||||
@@ -0,0 +1,218 @@
|
||||
#include "core/transformer.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ---- 微型算子 ---- */
|
||||
|
||||
/* y = x / sqrt(mean(x^2)+eps) * w */
|
||||
static void rmsnorm(const float *x, const float *w, float eps, int d, float *y) {
|
||||
float sq = 0.0f;
|
||||
for (int i = 0; i < d; i++) sq += x[i] * x[i];
|
||||
float inv = 1.0f / sqrtf(sq / (float)d + eps);
|
||||
for (int i = 0; i < d; i++) y[i] = x[i] * inv * w[i];
|
||||
}
|
||||
|
||||
/* y[cols] = x[k] @ W[k,cols]（行主序 W） */
|
||||
static void matvec(const float *x, const float *W, int k, int cols, float *y) {
|
||||
for (int j = 0; j < cols; j++) {
|
||||
float s = 0.0f;
|
||||
for (int i = 0; i < k; i++) s += x[i] * W[i * cols + j];
|
||||
y[j] = s;
|
||||
}
|
||||
}
|
||||
|
||||
/* GELU (tanh 近似)，与 Python 一致 */
|
||||
static void gelu(float *x, int n) {
|
||||
const float c = 0.7978845608028654f;
|
||||
const float a = 0.044715f;
|
||||
for (int i = 0; i < n; i++) {
|
||||
float v = x[i];
|
||||
x[i] = 0.5f * v * (1.0f + tanhf(c * (v + a * v * v * v)));
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- KV cache ---- */
|
||||
|
||||
void kv_init(KVState *kv, const Model *m) {
|
||||
memset(kv, 0, sizeof(*kv));
|
||||
kv->max_seq = m->config.max_seq_len;
|
||||
kv->n_layer = m->config.n_layer;
|
||||
size_t per = (size_t)kv->max_seq * m->config.n_head * m->config.head_dim;
|
||||
kv->k_cache = calloc((size_t)kv->n_layer, sizeof(float *));
|
||||
kv->v_cache = calloc((size_t)kv->n_layer, sizeof(float *));
|
||||
for (int l = 0; l < kv->n_layer; l++) {
|
||||
kv->k_cache[l] = malloc(per * sizeof(float));
|
||||
kv->v_cache[l] = malloc(per * sizeof(float));
|
||||
}
|
||||
kv->seq_len = 0;
|
||||
}
|
||||
|
||||
void kv_free(KVState *kv) {
|
||||
if (!kv) return;
|
||||
for (int l = 0; l < kv->n_layer; l++) {
|
||||
free(kv->k_cache[l]);
|
||||
free(kv->v_cache[l]);
|
||||
}
|
||||
free(kv->k_cache);
|
||||
free(kv->v_cache);
|
||||
memset(kv, 0, sizeof(*kv));
|
||||
}
|
||||
|
||||
void kv_reset(KVState *kv) { kv->seq_len = 0; }
|
||||
|
||||
void moe_embed(const Model *m, int token, int pos, float *x) {
|
||||
int d = m->config.d_model;
|
||||
const float *te = m->wte + (size_t)token * d;
|
||||
const float *pe = m->wpe + (size_t)pos * d;
|
||||
for (int i = 0; i < d; i++) x[i] = te[i] + pe[i];
|
||||
}
|
||||
|
||||
/* ---- 稠密 FFN：x -> gelu(x@w1) @ w2 ---- */
|
||||
static void dense_ffn(const Model *m, int l, const float *x, float *out) {
|
||||
int d = m->config.d_model;
|
||||
int f = m->config.d_ff;
|
||||
float *mid = malloc((size_t)f * sizeof(float));
|
||||
matvec(x, m->ffn_w1[l], d, f, mid);
|
||||
gelu(mid, f);
|
||||
matvec(mid, m->ffn_w2[l], f, d, out);
|
||||
free(mid);
|
||||
}
|
||||
|
||||
/* ---- MoE：router softmax + top-k 专家加权和 ---- */
|
||||
static void moe_ffn(const Model *m, int l, const float *x, float *out) {
|
||||
int d = m->config.d_model;
|
||||
int ne = m->config.moe_n_experts;
|
||||
int topk = m->config.moe_top_k;
|
||||
int de = m->config.d_expert;
|
||||
|
||||
/* router logits & softmax */
|
||||
float *probs = malloc((size_t)ne * sizeof(float));
|
||||
matvec(x, m->router[l], d, ne, probs);
|
||||
|
||||
float mx = probs[0];
|
||||
for (int e = 1; e < ne; e++) if (probs[e] > mx) mx = probs[e];
|
||||
float sum = 0.0f;
|
||||
for (int e = 0; e < ne; e++) { probs[e] = expf(probs[e] - mx); sum += probs[e]; }
|
||||
float inv = 1.0f / sum;
|
||||
for (int e = 0; e < ne; e++) probs[e] *= inv;
|
||||
|
||||
/* top-k 选择（在概率上做简单选择）*/
|
||||
int *idx = malloc((size_t)topk * sizeof(int));
|
||||
for (int t = 0; t < topk; t++) {
|
||||
int best = -1;
|
||||
float bestp = -1.0f;
|
||||
for (int e = 0; e < ne; e++) {
|
||||
int taken = 0;
|
||||
for (int q = 0; q < t; q++) if (idx[q] == e) { taken = 1; break; }
|
||||
if (!taken && probs[e] > bestp) { bestp = probs[e]; best = e; }
|
||||
}
|
||||
idx[t] = best;
|
||||
}
|
||||
|
||||
float *mid = malloc((size_t)de * sizeof(float));
|
||||
float *eo = malloc((size_t)d * sizeof(float));
|
||||
for (int i = 0; i < d; i++) out[i] = 0.0f;
|
||||
for (int t = 0; t < topk; t++) {
|
||||
int e = idx[t];
|
||||
matvec(x, m->exp_w1[l][e], d, de, mid);
|
||||
gelu(mid, de);
|
||||
matvec(mid, m->exp_w2[l][e], de, d, eo);
|
||||
float w = probs[e];
|
||||
for (int i = 0; i < d; i++) out[i] += w * eo[i];
|
||||
}
|
||||
free(probs);
|
||||
free(idx);
|
||||
free(mid);
|
||||
free(eo);
|
||||
}
|
||||
|
||||
/* ---- 单位置前向 ---- */
|
||||
void moe_forward(const Model *m, KVState *kv, int pos, const float *x, float *logits) {
|
||||
const ModelConfig *c = &m->config;
|
||||
int d = c->d_model, nh = c->n_head, hd = c->head_dim;
|
||||
int L = c->n_layer, V = c->vocab_size;
|
||||
|
||||
float *a = malloc((size_t)d * sizeof(float)); /* norm out */
|
||||
float *q = malloc((size_t)(nh * hd) * sizeof(float));
|
||||
float *k = malloc((size_t)(nh * hd) * sizeof(float));
|
||||
float *v = malloc((size_t)(nh * hd) * sizeof(float));
|
||||
float *att = malloc((size_t)d * sizeof(float)); /* 注意力输出（暂存 o 前） */
|
||||
float *o = malloc((size_t)d * sizeof(float));
|
||||
float *ffn = malloc((size_t)d * sizeof(float));
|
||||
float *scores = malloc((size_t)(kv->max_seq) * sizeof(float));
|
||||
|
||||
float *h = malloc((size_t)d * sizeof(float));
|
||||
memcpy(h, x, (size_t)d * sizeof(float));
|
||||
|
||||
float inv_sqrt = 1.0f / sqrtf((float)hd);
|
||||
|
||||
for (int l = 0; l < L; l++) {
|
||||
/* 1) norm1 */
|
||||
rmsnorm(h, m->l_norm1[l], c->rmsnorm_eps, d, a);
|
||||
|
||||
/* 2) qkv */
|
||||
matvec(a, m->wq[l], d, d, q);
|
||||
matvec(a, m->wk[l], d, d, k);
|
||||
matvec(a, m->wv[l], d, d, v);
|
||||
|
||||
/* 存 cache：index = pos*(nh*hd) + h*hd + t */
|
||||
{
|
||||
float *kc = kv->k_cache[l] + (size_t)pos * nh * hd;
|
||||
float *vc = kv->v_cache[l] + (size_t)pos * nh * hd;
|
||||
memcpy(kc, k, (size_t)(nh * hd) * sizeof(float));
|
||||
memcpy(vc, v, (size_t)(nh * hd) * sizeof(float));
|
||||
}
|
||||
|
||||
/* 3) 自注意力（因果），每个 head 独立 */
|
||||
for (int hh = 0; hh < nh; hh++) {
|
||||
const float *qh = q + (size_t)hh * hd;
|
||||
/* scores[j] = (qh . k_j)/sqrt(hd) for j in 0..pos */
|
||||
for (int j = 0; j <= pos; j++) {
|
||||
const float *kj = kv->k_cache[l] + (size_t)j * nh * hd + (size_t)hh * hd;
|
||||
float s = 0.0f;
|
||||
for (int t = 0; t < hd; t++) s += qh[t] * kj[t];
|
||||
scores[j] = s * inv_sqrt;
|
||||
}
|
||||
float mxs = scores[0];
|
||||
for (int j = 1; j <= pos; j++) if (scores[j] > mxs) mxs = scores[j];
|
||||
float ss = 0.0f;
|
||||
for (int j = 0; j <= pos; j++) { scores[j] = expf(scores[j] - mxs); ss += scores[j]; }
|
||||
float is = 1.0f / ss;
|
||||
for (int j = 0; j <= pos; j++) scores[j] *= is;
|
||||
|
||||
for (int t = 0; t < hd; t++) {
|
||||
float acc = 0.0f;
|
||||
for (int j = 0; j <= pos; j++) {
|
||||
const float *vj = kv->v_cache[l] + (size_t)j * nh * hd + (size_t)hh * hd;
|
||||
acc += scores[j] * vj[t];
|
||||
}
|
||||
att[(size_t)hh * hd + t] = acc;
|
||||
}
|
||||
}
|
||||
|
||||
/* 4) o = att @ wo */
|
||||
matvec(att, m->wo[l], d, d, o);
|
||||
/* 5) residual */
|
||||
for (int i = 0; i < d; i++) h[i] += o[i];
|
||||
|
||||
/* 6) norm2 */
|
||||
rmsnorm(h, m->l_norm2[l], c->rmsnorm_eps, d, a);
|
||||
|
||||
/* 7) ffn / moe */
|
||||
if (c->moe_mask[l]) {
|
||||
moe_ffn(m, l, a, ffn);
|
||||
} else {
|
||||
dense_ffn(m, l, a, ffn);
|
||||
}
|
||||
for (int i = 0; i < d; i++) h[i] += ffn[i];
|
||||
}
|
||||
|
||||
/* 最终 norm + lm_head */
|
||||
rmsnorm(h, m->norm_final, c->rmsnorm_eps, d, a);
|
||||
matvec(a, m->lm_head, d, V, logits); /* lm_head 为 (d, vocab) 行主序 */
|
||||
|
||||
free(a); free(q); free(k); free(v); free(att); free(o);
|
||||
free(ffn); free(scores); free(h);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
#ifndef MO_TRANSFORMER_H
|
||||
#define MO_TRANSFORMER_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "model/model.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* KV 缓存：每层一份，容量 = max_seq * n_head * head_dim */
|
||||
typedef struct {
|
||||
int seq_len; /* 已处理位置数 */
|
||||
int max_seq;
|
||||
int n_layer;
|
||||
float **k_cache; /* [layer] */
|
||||
float **v_cache; /* [layer] */
|
||||
} KVState;
|
||||
|
||||
void kv_init(KVState *kv, const Model *m);
|
||||
void kv_free(KVState *kv);
|
||||
void kv_reset(KVState *kv);
|
||||
|
||||
/* 计算单个位置 token 的嵌入 x = wte[token] + wpe[pos]，写入 x(d_model) */
|
||||
void moe_embed(const Model *m, int token, int pos, float *x);
|
||||
|
||||
/* 对位置 pos 执行一次前向，返回该位置的 logits(vocab)。
|
||||
会写入 KV cache（pos 处）。调用方负责已就绪的嵌入与 cache 状态。 */
|
||||
void moe_forward(const Model *m, KVState *kv, int pos, const float *x, float *logits);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_TRANSFORMER_H */
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
#include "infer/generate.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "infer/sample.h"
|
||||
#include "model/tokenizer.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
/* ---- 流式生成 ---- */
|
||||
|
||||
int gen_stream_begin(GenStream *gs, const Model *m, const char *prompt,
|
||||
int max_new, float temperature, int top_k) {
|
||||
memset(gs, 0, sizeof(*gs));
|
||||
gs->m = m;
|
||||
gs->max_new = max_new;
|
||||
gs->temperature = temperature;
|
||||
gs->top_k = top_k;
|
||||
|
||||
const int max_seq = m->config.max_seq_len;
|
||||
gs->ids = malloc((size_t)(max_seq + 16) * sizeof(int));
|
||||
if (!gs->ids) return -1;
|
||||
gs->ptoks = tokenizer_encode(&m->tok, prompt, gs->ids, max_seq + 16);
|
||||
if (gs->ptoks < 0) { free(gs->ids); gs->ids = NULL; return -1; }
|
||||
if (gs->ptoks > max_seq) gs->ptoks = max_seq;
|
||||
|
||||
gs->logits = malloc((size_t)m->config.vocab_size * sizeof(float));
|
||||
gs->x = malloc((size_t)m->config.d_model * sizeof(float));
|
||||
if (!gs->logits || !gs->x) return -1;
|
||||
kv_init(&gs->kv, m);
|
||||
|
||||
/* 预填充 prompt，建立 KV cache */
|
||||
for (int i = 0; i < gs->ptoks; i++) {
|
||||
moe_embed(m, gs->ids[i], i, gs->x);
|
||||
moe_forward(m, &gs->kv, i, gs->x, gs->logits);
|
||||
}
|
||||
gs->pos = gs->ptoks;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int gen_stream_next(GenStream *gs, char *out_buf, size_t out_sz) {
|
||||
const Model *m = gs->m;
|
||||
if (gs->done) return 0;
|
||||
if (gs->emitted >= gs->max_new || gs->pos >= m->config.max_seq_len) {
|
||||
gs->done = 1;
|
||||
return 0;
|
||||
}
|
||||
int next = sample_top_k(gs->logits, m->config.vocab_size, gs->temperature, gs->top_k);
|
||||
tokenizer_decode(&m->tok, &next, 1, out_buf, (int)out_sz);
|
||||
|
||||
moe_embed(m, next, gs->pos, gs->x);
|
||||
moe_forward(m, &gs->kv, gs->pos, gs->x, gs->logits);
|
||||
gs->pos++;
|
||||
gs->emitted++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void gen_stream_end(GenStream *gs) {
|
||||
if (!gs) return;
|
||||
kv_free(&gs->kv);
|
||||
free(gs->logits);
|
||||
free(gs->x);
|
||||
free(gs->ids);
|
||||
memset(gs, 0, sizeof(*gs));
|
||||
}
|
||||
|
||||
/* ---- 一次性生成（CLI） ---- */
|
||||
|
||||
int moe_generate(const Model *m, const char *prompt, int max_new_tokens,
|
||||
float temperature, int top_k) {
|
||||
GenStream gs;
|
||||
if (gen_stream_begin(&gs, m, prompt, max_new_tokens, temperature, top_k) < 0) {
|
||||
MO_LOGE("prompt 编码失败");
|
||||
return -1;
|
||||
}
|
||||
printf("%s", prompt);
|
||||
fflush(stdout);
|
||||
char buf[64];
|
||||
while (gen_stream_next(&gs, buf, sizeof(buf)) == 1) {
|
||||
printf("%s", buf);
|
||||
fflush(stdout);
|
||||
}
|
||||
printf("\n");
|
||||
gen_stream_end(&gs);
|
||||
return 0;
|
||||
}
|
||||
|
||||
char *moe_generate_text(const Model *m, const char *prompt, int max_new_tokens,
|
||||
float temperature, int top_k) {
|
||||
GenStream gs;
|
||||
size_t cap = strlen(prompt) + (size_t)max_new_tokens * 16 + 128;
|
||||
char *buf = malloc(cap);
|
||||
if (!buf) return NULL;
|
||||
size_t w = (size_t)snprintf(buf, cap, "%s", prompt);
|
||||
|
||||
if (gen_stream_begin(&gs, m, prompt, max_new_tokens, temperature, top_k) < 0) {
|
||||
MO_LOGE("prompt 编码失败");
|
||||
gen_stream_end(&gs);
|
||||
return buf;
|
||||
}
|
||||
char tok[64];
|
||||
while (gen_stream_next(&gs, tok, sizeof(tok)) == 1) {
|
||||
size_t tl = strlen(tok);
|
||||
if (w + tl + 1 > cap) {
|
||||
cap = cap * 2 + tl + 64;
|
||||
buf = realloc(buf, cap);
|
||||
}
|
||||
memcpy(buf + w, tok, tl);
|
||||
w += tl;
|
||||
buf[w] = '\0';
|
||||
}
|
||||
gen_stream_end(&gs);
|
||||
return buf;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
#ifndef MO_GENERATE_H
|
||||
#define MO_GENERATE_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "core/transformer.h"
|
||||
#include "model/model.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---------- 一次性生成（CLI 用，自回归打印到 stdout） ---------- */
|
||||
int moe_generate(const Model *m, const char *prompt, int max_new_tokens,
|
||||
float temperature, int top_k);
|
||||
|
||||
/* 生成并返回完整文本（prompt + 生成）。调用方 free()。失败返回 strdup(prompt)。 */
|
||||
char *moe_generate_text(const Model *m, const char *prompt, int max_new_tokens,
|
||||
float temperature, int top_k);
|
||||
|
||||
/* ---------- 流式生成（逐个 token 拉取，供 HTTP/SSE 用） ---------- */
|
||||
typedef struct {
|
||||
const Model *m;
|
||||
KVState kv; /* 内部 KV cache */
|
||||
float *logits;
|
||||
float *x;
|
||||
int *ids; /* prompt token 缓存 */
|
||||
int ptoks;
|
||||
int pos;
|
||||
int emitted; /* 已产出 token 数 */
|
||||
int max_new;
|
||||
float temperature;
|
||||
int top_k;
|
||||
int done;
|
||||
char last_buf[64]; /* 最后一个 token 的utf8（避免重复解码） */
|
||||
} GenStream;
|
||||
|
||||
/* 初始化 + 预填充 prompt。成功返回 0 */
|
||||
int gen_stream_begin(GenStream *gs, const Model *m, const char *prompt,
|
||||
int max_new, float temperature, int top_k);
|
||||
/* 拉取下一个 token 文本到 out_buf；返回 1=产出tokens，0=结束，负=出错 */
|
||||
int gen_stream_next(GenStream *gs, char *out_buf, size_t out_sz);
|
||||
void gen_stream_end(GenStream *gs);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_GENERATE_H */
|
||||
@@ -0,0 +1,64 @@
|
||||
#include "infer/sample.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
static uint32_t g_rng = 2463534242u;
|
||||
|
||||
void mo_rng_seed(unsigned seed) { g_rng = seed ? seed : 2463534242u; }
|
||||
|
||||
static uint32_t next_rng(void) {
|
||||
uint32_t x = g_rng;
|
||||
x ^= x << 13;
|
||||
x ^= x >> 17;
|
||||
x ^= x << 5;
|
||||
g_rng = x;
|
||||
return x;
|
||||
}
|
||||
|
||||
typedef struct { float v; int idx; } Scored;
|
||||
|
||||
static int cmp_desc(const void *a, const void *b) {
|
||||
const Scored *pa = (const Scored *)a, *pb = (const Scored *)b;
|
||||
if (pa->v > pb->v) return -1;
|
||||
if (pa->v < pb->v) return 1;
|
||||
return pa->idx - pb->idx;
|
||||
}
|
||||
|
||||
int sample_top_k(const float *logits, int vocab, float temperature, int top_k) {
|
||||
if (temperature <= 0.0f) {
|
||||
/* 贪心 */
|
||||
int best = 0;
|
||||
for (int i = 1; i < vocab; i++) if (logits[i] > logits[best]) best = i;
|
||||
return best;
|
||||
}
|
||||
if (top_k <= 0 || top_k > vocab) top_k = vocab;
|
||||
|
||||
Scored *sc = malloc((size_t)vocab * sizeof(Scored));
|
||||
for (int i = 0; i < vocab; i++) {
|
||||
sc[i].v = logits[i] / temperature;
|
||||
sc[i].idx = i;
|
||||
}
|
||||
qsort(sc, (size_t)vocab, sizeof(Scored), cmp_desc);
|
||||
|
||||
/* softmax over 前 top_k 个 */
|
||||
float mxs = sc[0].v;
|
||||
float sum = 0.0f;
|
||||
for (int i = 0; i < top_k; i++) {
|
||||
sc[i].v = expf(sc[i].v - mxs);
|
||||
sum += sc[i].v;
|
||||
}
|
||||
float target = ((float)(next_rng() & 0xFFFFFFu) / (float)(1u << 24)) * sum;
|
||||
float acc = 0.0f;
|
||||
for (int i = 0; i < top_k; i++) {
|
||||
acc += sc[i].v;
|
||||
if (target <= acc) {
|
||||
int idx = sc[i].idx;
|
||||
free(sc);
|
||||
return idx;
|
||||
}
|
||||
}
|
||||
int idx = sc[top_k - 1].idx;
|
||||
free(sc);
|
||||
return idx;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#ifndef MO_SAMPLE_H
|
||||
#define MO_SAMPLE_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
void mo_rng_seed(unsigned seed);
|
||||
|
||||
/* 依据 logits 采样一个 token id。
|
||||
temperature 缩放 logits；top_k 限制候选；用可复现的 xorshift。 */
|
||||
int sample_top_k(const float *logits, int vocab, float temperature, int top_k);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_SAMPLE_H */
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "model/llama.h"
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "用法: pmai-llama <model.gguf> [n_new] [prompt 或 init_id ...]\n");
|
||||
return 1;
|
||||
}
|
||||
LlamaModel m;
|
||||
if (llama_load(&m, argv[1]) != 0) {
|
||||
fprintf(stderr, "加载失败: %s\n", argv[1]);
|
||||
return 2;
|
||||
}
|
||||
int n_new = argc > 2 ? atoi(argv[2]) : 10;
|
||||
int ids[512];
|
||||
int n = 0;
|
||||
|
||||
/* --tok <text>：仅分词，打印 token id 后退出（用于验证分词器） */
|
||||
if (m.is_loaded && m.tok.vocab_size > 0 && argc >= 5 && strcmp(argv[3], "--tok") == 0) {
|
||||
n = llama_tokenize(&m, argv[4], ids, 512);
|
||||
for (int i = 0; i < n; i++) printf("%d", ids[i]), putchar(i + 1 < n ? ' ' : '\n');
|
||||
llama_free(&m);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 有分词器且第 3 个参数是字符串 -> 当作 prompt 编码 */
|
||||
if (m.is_loaded && m.tok.vocab_size > 0 && argc >= 4) {
|
||||
n = llama_tokenize(&m, argv[3], ids, 512);
|
||||
if (n <= 0) { ids[0] = 1; n = 1; }
|
||||
} else {
|
||||
for (int i = 3; i < argc && n < 512; i++) ids[n++] = atoi(argv[i]);
|
||||
if (n == 0) ids[n++] = 1;
|
||||
}
|
||||
|
||||
int out[512];
|
||||
int p = llama_generate(&m, ids, n, n_new, 0.0f, 0, out);
|
||||
|
||||
if (m.tok.vocab_size > 0) {
|
||||
char buf[4096];
|
||||
int blen = llama_detokenize(&m, out, p, buf, sizeof(buf));
|
||||
fwrite(buf, 1, (size_t)(blen < 0 ? 0 : blen), stdout);
|
||||
printf("\n");
|
||||
} else {
|
||||
for (int i = 0; i < p; i++) printf("%d", out[i]), putchar(i + 1 < p ? ' ' : '\n');
|
||||
}
|
||||
llama_free(&m);
|
||||
return 0;
|
||||
}
|
||||
+275
@@ -0,0 +1,275 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "infer/generate.h"
|
||||
#include "infer/sample.h"
|
||||
#include "model/gguf.h"
|
||||
#include "model/model.h"
|
||||
#ifdef MOE_HTTP
|
||||
#include "server/api.h"
|
||||
#endif
|
||||
#include "utils/logger.h"
|
||||
|
||||
static void usage(void) {
|
||||
printf("用法: pmai --model <file.pap|file.gguf> [选项]\n"
|
||||
" --model <path> 模型文件（必填；.gguf 自动识别）\n"
|
||||
" --inspect <path> 查看任意 GGUF 的元数据/张量表\n"
|
||||
" --prompt <text> 提示词（默认: \"The\"）\n"
|
||||
" --n_tokens <N> 生成 token 数（默认 100）\n"
|
||||
" --temperature <T> 采样温度（默认 0.8，<=0 为贪心）\n"
|
||||
" --top_k <K> top-k 采样（默认 40，<=0 不限制）\n"
|
||||
" --seed <S> 随机种子\n"
|
||||
" --interactive 交互式对话（输入一行→生成→继续，exit/quit 退出）\n"
|
||||
" --chat 聊天模式（User:/Assistant: 格式，带上下文，存记录）\n"
|
||||
" --output <file> 把生成文本写到文件（默认在 output/ 目录）\n"
|
||||
" --serve 以 HTTP 服务方式运行（需 --model）\n"
|
||||
" --port <P> HTTP 端口（默认 11434）\n"
|
||||
" --log-level <lvl> debug/info/warn/error\n");
|
||||
}
|
||||
|
||||
static int has_gguf_magic(const char *path) {
|
||||
const char m[4] = {'G', 'G', 'U', 'F'};
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) return 0;
|
||||
char b[4];
|
||||
int n = (int)fread(b, 1, 4, f);
|
||||
fclose(f);
|
||||
return (n == 4 && memcmp(b, m, 4) == 0);
|
||||
}
|
||||
|
||||
static int arg_i(const char *s, int def) { return s ? atoi(s) : def; }
|
||||
static float arg_f(const char *s, float def) { return s ? (float)atof(s) : def; }
|
||||
|
||||
static int write_text_file(const char *path, const char *content) {
|
||||
FILE *f = fopen(path, "wb");
|
||||
if (!f) return -1;
|
||||
fwrite(content, 1, strlen(content), f);
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 从简单 key: value 的 config 文件加载默认值（host/port/model/n_tokens/temperature/top_k） */
|
||||
static void load_config(const char *path, unsigned *port, int *n_tokens, float *temperature,
|
||||
int *top_k, char *model, size_t model_cap) {
|
||||
FILE *f = fopen(path, "r");
|
||||
if (!f) return;
|
||||
char line[512];
|
||||
while (fgets(line, sizeof(line), f)) {
|
||||
char *c = strchr(line, '#');
|
||||
if (c) *c = '\0';
|
||||
if ((line[0] == '\n' || line[0] == '\r') && !line[1]) continue;
|
||||
char key[160] = {0}, val[384] = {0};
|
||||
if (sscanf(line, "%159[^:]: %383[^\n]", key, val) != 2) continue;
|
||||
for (char *p = key; *p; p++) if (*p == ' ' || *p == '\t') { *p = '\0'; break; }
|
||||
if (!strcmp(key, "port")) *port = (unsigned)atoi(val);
|
||||
else if (!strcmp(key, "model")) snprintf(model, model_cap, "%s", val);
|
||||
else if (!strcmp(key, "n_tokens")) *n_tokens = atoi(val);
|
||||
else if (!strcmp(key, "temperature")) *temperature = (float)atof(val);
|
||||
else if (!strcmp(key, "top_k")) *top_k = atoi(val);
|
||||
}
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
/* 聊天式生成：从 prompt 前缀开始生成，遇到下一个 "\nUser:"（下一轮用户话）停止，返回助手回复。调用方 free() */
|
||||
static char *run_chat(Model *m, const char *prompt, int max_new, float temperature, int top_k) {
|
||||
GenStream gs;
|
||||
if (gen_stream_begin(&gs, m, prompt, max_new, temperature, top_k) < 0) return strdup("");
|
||||
size_t cap = (size_t)max_new * 16 + 256;
|
||||
char *buf = malloc(cap);
|
||||
size_t w = 0;
|
||||
buf[0] = '\0';
|
||||
char tok[64];
|
||||
while (gen_stream_next(&gs, tok, sizeof(tok)) == 1) {
|
||||
size_t tl = strlen(tok);
|
||||
if (w + tl + 1 > cap) {
|
||||
cap = cap * 2 + tl + 64;
|
||||
buf = realloc(buf, cap);
|
||||
}
|
||||
memcpy(buf + w, tok, tl);
|
||||
w += tl;
|
||||
buf[w] = '\0';
|
||||
if (strstr(buf, "\nUser:")) break; /* 已生成到下一轮用户话，停止 */
|
||||
}
|
||||
gen_stream_end(&gs);
|
||||
char *np = strstr(buf, "\nUser:");
|
||||
if (np) *np = '\0'; /* 截掉后面的下一轮 */
|
||||
return buf;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
const char *model = NULL, *prompt = NULL, *loglvl = NULL, *inspect = NULL;
|
||||
int n_tokens = 100;
|
||||
float temperature = 0.8f;
|
||||
int top_k = 40;
|
||||
int seed = 0;
|
||||
int serve = 0;
|
||||
int interactive = 0;
|
||||
int chat = 0;
|
||||
unsigned port = 11434;
|
||||
const char *output = NULL;
|
||||
const char *config = NULL;
|
||||
char config_model[512] = "";
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!strcmp(argv[i], "--model") && i + 1 < argc) model = argv[++i];
|
||||
else if (!strcmp(argv[i], "--inspect") && i + 1 < argc) inspect = argv[++i];
|
||||
else if (!strcmp(argv[i], "--prompt") && i + 1 < argc) prompt = argv[++i];
|
||||
else if (!strcmp(argv[i], "--n_tokens") && i + 1 < argc) n_tokens = arg_i(argv[++i], 100);
|
||||
else if (!strcmp(argv[i], "--temperature") && i + 1 < argc) temperature = arg_f(argv[++i], 0.8f);
|
||||
else if (!strcmp(argv[i], "--top_k") && i + 1 < argc) top_k = arg_i(argv[++i], 40);
|
||||
else if (!strcmp(argv[i], "--seed") && i + 1 < argc) seed = arg_i(argv[++i], 0);
|
||||
else if (!strcmp(argv[i], "--interactive")) interactive = 1;
|
||||
else if (!strcmp(argv[i], "--chat")) chat = 1;
|
||||
else if (!strcmp(argv[i], "--output") && i + 1 < argc) output = argv[++i];
|
||||
else if (!strcmp(argv[i], "--config") && i + 1 < argc) config = argv[++i];
|
||||
else if (!strcmp(argv[i], "--serve")) serve = 1;
|
||||
else if (!strcmp(argv[i], "--port") && i + 1 < argc) port = (unsigned)arg_i(argv[++i], 11434);
|
||||
else if (!strcmp(argv[i], "--log-level") && i + 1 < argc) loglvl = argv[++i];
|
||||
else if (!strcmp(argv[i], "--help") || !strcmp(argv[i], "-h")) { usage(); return 0; }
|
||||
}
|
||||
|
||||
/* 便利：未用 --model 时，把第一个非 '-' 位置参数当作模型路径 */
|
||||
if (config) {
|
||||
load_config(config, &port, &n_tokens, &temperature, &top_k, config_model, sizeof(config_model));
|
||||
if (!model && config_model[0]) model = config_model;
|
||||
}
|
||||
if (!model && argc > 1 && argv[1][0] != '-') model = argv[1];
|
||||
|
||||
if (loglvl) {
|
||||
if (!strcasecmp(loglvl, "debug")) mo_log_set_level(MO_LOG_DEBUG);
|
||||
else if (!strcasecmp(loglvl, "info")) mo_log_set_level(MO_LOG_INFO);
|
||||
else if (!strcasecmp(loglvl, "warn")) mo_log_set_level(MO_LOG_WARN);
|
||||
else if (!strcasecmp(loglvl, "error")) mo_log_set_level(MO_LOG_ERROR);
|
||||
}
|
||||
|
||||
if (inspect) {
|
||||
Gguf g;
|
||||
if (gguf_open(&g, inspect) != 0) { MO_LOGE("GGUF 打开失败: %s", inspect); return 2; }
|
||||
gguf_inspect(&g, inspect, stdout);
|
||||
gguf_close(&g);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!model) { usage(); return 1; }
|
||||
|
||||
mo_rng_seed((unsigned)seed);
|
||||
|
||||
Model m;
|
||||
int rc = has_gguf_magic(model) ? model_load_gguf(&m, model) : model_load(&m, model);
|
||||
if (rc != 0) {
|
||||
MO_LOGE("加载模型失败(%d): %s", rc, model);
|
||||
return 2;
|
||||
}
|
||||
ModelConfig *c = &m.config;
|
||||
MO_LOGI("已加载模型: %s", model);
|
||||
MO_LOGI(" config: vocab=%d layers=%d d_model=%d heads=%d max_seq=%d",
|
||||
c->vocab_size, c->n_layer, c->d_model, c->n_head, c->max_seq_len);
|
||||
MO_LOGI(" MoE: experts=%d top_k=%d d_expert=%d",
|
||||
c->moe_n_experts, c->moe_top_k, c->d_expert);
|
||||
MO_LOGI(" params: %lld", (long long)model_param_count(&m));
|
||||
|
||||
if (!prompt) prompt = "The";
|
||||
MO_LOGI("生成中: n_tokens=%d temperature=%.2f top_k=%d", n_tokens, temperature, top_k);
|
||||
|
||||
if (serve) {
|
||||
#ifdef MOE_HTTP
|
||||
moe_server_run(&m, model, port);
|
||||
#else
|
||||
MO_LOGE("未启用 HTTP 服务（缺 libmicrohttpd/jansson，用 -DBUILD_HTTP=ON 重建）");
|
||||
(void)port;
|
||||
#endif
|
||||
model_free(&m);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (chat) {
|
||||
mkdir("output", 0755);
|
||||
char outpath[512];
|
||||
if (output) snprintf(outpath, sizeof(outpath), "%s", output);
|
||||
else snprintf(outpath, sizeof(outpath), "output/chat-%ld.txt", (long)time(NULL));
|
||||
FILE *logf = fopen(outpath, "wb");
|
||||
if (logf) fprintf(logf, "# pmai 聊天记录\n\n");
|
||||
MO_LOGI("聊天模式已启动。输入内容即对话；exit/quit 退出。记录保存到 %s", outpath);
|
||||
|
||||
size_t hcap = 4096, hlen = 0;
|
||||
char *hist = calloc(1, hcap);
|
||||
char line[4096];
|
||||
for (;;) {
|
||||
printf("\n你: ");
|
||||
fflush(stdout);
|
||||
if (!fgets(line, sizeof(line), stdin)) break;
|
||||
size_t len = strlen(line);
|
||||
while (len && (line[len - 1] == '\n' || line[len - 1] == '\r')) line[--len] = '\0';
|
||||
if (!len) continue;
|
||||
if (!strcmp(line, "exit") || !strcmp(line, "quit")) break;
|
||||
|
||||
/* prompt = history + "User: <line>\nAssistant:" */
|
||||
size_t need = hlen + len + 64;
|
||||
if (need > hcap) { while (need > hcap) hcap *= 2; hist = realloc(hist, hcap); }
|
||||
size_t p = (size_t)snprintf(hist + hlen, hcap - hlen, "User: %s\nAssistant:", line);
|
||||
hlen += p;
|
||||
|
||||
char *reply = run_chat(&m, hist, n_tokens, temperature, top_k);
|
||||
if (!reply || !reply[0]) { free(reply); reply = strdup("(我还没想好，换句话试试？)"); }
|
||||
printf("%s\n", reply);
|
||||
|
||||
/* 回填历史，供下一轮携带上下文；保留尾部避免超出上下文 */
|
||||
size_t adds = (size_t)snprintf(hist + hlen, hcap - hlen, "\nAssistant: %s\n", reply ? reply : "");
|
||||
hlen += adds;
|
||||
if (hlen > 2048) { size_t keep = 1536; memmove(hist, hist + hlen - keep, keep + 1); hlen = keep; }
|
||||
if (logf && reply) { fprintf(logf, "User: %s\nAssistant: %s\n\n", line, reply); fflush(logf); }
|
||||
free(reply);
|
||||
}
|
||||
free(hist);
|
||||
if (logf) fclose(logf);
|
||||
model_free(&m);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (interactive) {
|
||||
mkdir("output", 0755);
|
||||
char outpath[512];
|
||||
if (output) snprintf(outpath, sizeof(outpath), "%s", output);
|
||||
else snprintf(outpath, sizeof(outpath), "output/chat-%ld.txt", (long)time(NULL));
|
||||
FILE *logf = fopen(outpath, "wb");
|
||||
if (logf) fprintf(logf, "# pmai 交互记录\n\n");
|
||||
MO_LOGI("交互模式已启动。输入一行后回车即生成；输入 exit/quit 退出。对话保存到 %s", outpath);
|
||||
char line[4096];
|
||||
for (;;) {
|
||||
printf("\n>> ");
|
||||
fflush(stdout);
|
||||
if (!fgets(line, sizeof(line), stdin)) break;
|
||||
size_t len = strlen(line);
|
||||
while (len && (line[len - 1] == '\n' || line[len - 1] == '\r')) line[--len] = '\0';
|
||||
if (!len) continue;
|
||||
if (!strcmp(line, "exit") || !strcmp(line, "quit")) break;
|
||||
char *out = moe_generate_text(&m, line, n_tokens, temperature, top_k);
|
||||
printf("%s\n", out ? out : "");
|
||||
if (logf && out) { fprintf(logf, ">> %s\n%s\n\n", line, out); fflush(logf); }
|
||||
free(out);
|
||||
}
|
||||
if (logf) fclose(logf);
|
||||
model_free(&m);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (output) {
|
||||
mkdir("output", 0755);
|
||||
char *out = moe_generate_text(&m, prompt, n_tokens, temperature, top_k);
|
||||
int rc = write_text_file(output, out ? out : "");
|
||||
MO_LOGI("生成已写入 %s (%s)", output, rc == 0 ? "ok" : "fail");
|
||||
printf("%s\n", out ? out : "");
|
||||
free(out);
|
||||
model_free(&m);
|
||||
return 0;
|
||||
}
|
||||
|
||||
moe_generate(&m, prompt, n_tokens, temperature, top_k);
|
||||
|
||||
model_free(&m);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include "model/config.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
void config_init(ModelConfig *c) {
|
||||
memset(c, 0, sizeof(*c));
|
||||
c->head_dim = 0;
|
||||
c->rmsnorm_eps = 1e-5f;
|
||||
}
|
||||
|
||||
void config_free(ModelConfig *c) {
|
||||
if (!c) return;
|
||||
free(c->moe_mask);
|
||||
c->moe_mask = NULL;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#ifndef MO_CONFIG_H
|
||||
#define MO_CONFIG_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
int vocab_size;
|
||||
int n_layer;
|
||||
int d_model;
|
||||
int n_head;
|
||||
int head_dim; /* d_model / n_head */
|
||||
int d_ff; /* 稠密 FFN 中间层 */
|
||||
int moe_n_experts;
|
||||
int moe_top_k;
|
||||
int d_expert; /* MoE 专家中间层 */
|
||||
int max_seq_len;
|
||||
int num_merges;
|
||||
int n_special;
|
||||
int tie_weights;
|
||||
float rmsnorm_eps;
|
||||
unsigned char *moe_mask; /* n_layer: 1=MoE, 0=dense */
|
||||
} ModelConfig;
|
||||
|
||||
void config_init(ModelConfig *c);
|
||||
void config_free(ModelConfig *c);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_CONFIG_H */
|
||||
+465
@@ -0,0 +1,465 @@
|
||||
#include "model/gguf.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ---- ggml_type 枚举（常用） ---- */
|
||||
#define GGML_F32 0
|
||||
#define GGML_F16 1
|
||||
#define GGML_Q4_0 2
|
||||
#define GGML_Q5_0 6
|
||||
#define GGML_Q5_1 7
|
||||
#define GGML_Q8_0 8
|
||||
#define GGML_BF16 16
|
||||
|
||||
/* ---- 元数据值类型 ---- */
|
||||
#define GGVAL_UINT8 0
|
||||
#define GGVAL_INT8 1
|
||||
#define GGVAL_UINT16 2
|
||||
#define GGVAL_INT16 3
|
||||
#define GGVAL_UINT32 4
|
||||
#define GGVAL_INT32 5
|
||||
#define GGVAL_FLOAT32 6
|
||||
#define GGVAL_BOOL 7
|
||||
#define GGVAL_STRING 8
|
||||
#define GGVAL_ARRAY 9
|
||||
#define GGVAL_UINT64 10
|
||||
#define GGVAL_INT64 11
|
||||
#define GGVAL_FLOAT64 12
|
||||
|
||||
static const unsigned char *gd(const Gguf *g, size_t off) { return g->data + off; }
|
||||
|
||||
static uint64_t rd_u64(const unsigned char *p) {
|
||||
uint64_t v; memcpy(&v, p, 8); return v;
|
||||
}
|
||||
static uint32_t rd_u32(const unsigned char *p) {
|
||||
uint32_t v; memcpy(&v, p, 4); return v;
|
||||
}
|
||||
static uint16_t rd_u16(const unsigned char *p) {
|
||||
uint16_t v; memcpy(&v, p, 2); return v;
|
||||
}
|
||||
static float rd_f32(const unsigned char *p) {
|
||||
float v; memcpy(&v, p, 4); return v;
|
||||
}
|
||||
|
||||
static float bf16_to_f32(uint16_t h) {
|
||||
uint32_t f = (uint32_t)h << 16;
|
||||
float v; memcpy(&v, &f, 4); return v;
|
||||
}
|
||||
|
||||
static float fp16_to_f32(uint16_t h) {
|
||||
uint32_t sign = (uint32_t)(h >> 15) & 1u;
|
||||
uint32_t exp = (h >> 10) & 0x1fu;
|
||||
uint32_t mant = h & 0x3ffu;
|
||||
uint32_t bits;
|
||||
float f;
|
||||
if (exp == 0x1fu) { bits = (sign << 31) | 0x7f800000u | (mant << 13); memcpy(&f, &bits, 4); return f; }
|
||||
if (exp == 0) {
|
||||
if (mant == 0) { bits = sign << 31; memcpy(&f, &bits, 4); return f; }
|
||||
f = (float)mant * 5.9604644775390625e-8f;
|
||||
return sign ? -f : f;
|
||||
}
|
||||
bits = (sign << 31) | ((exp - 15 + 127) << 23) | (mant << 13);
|
||||
memcpy(&f, &bits, 4);
|
||||
return f;
|
||||
}
|
||||
|
||||
/* 块大小 / 每块字节数（llama.cpp ggml 类型；Q4_0 等 block=32） */
|
||||
static uint32_t type_block_bytes(uint32_t t) {
|
||||
switch (t) {
|
||||
case GGML_F32: return 32 * 4;
|
||||
case GGML_F16: return 32 * 2;
|
||||
case GGML_BF16: return 32 * 2;
|
||||
case GGML_Q8_0: return 2 + 32; /* d(f16) + 32*int8 */
|
||||
case GGML_Q4_0: return 2 + 16; /* d(f16) + 16*4bit(=32) */
|
||||
case GGML_Q5_0: return 2 + 16 + 4; /* d + low + qh */
|
||||
case GGML_Q5_1: return 2 + 2 + 16 + 4; /* d + m + low + qh */
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* 反量化一个块（32 个）到 out */
|
||||
static void dequant_block(uint32_t t, const unsigned char *p, float *out) {
|
||||
int i;
|
||||
switch (t) {
|
||||
case GGML_F32: {
|
||||
for (i = 0; i < 32; i++) out[i] = rd_f32(p + i * 4);
|
||||
break;
|
||||
}
|
||||
case GGML_F16: {
|
||||
for (i = 0; i < 32; i++) out[i] = fp16_to_f32(*(const uint16_t *)(p + i * 2));
|
||||
break;
|
||||
}
|
||||
case GGML_BF16: {
|
||||
for (i = 0; i < 32; i++) out[i] = bf16_to_f32(*(const uint16_t *)(p + i * 2));
|
||||
break;
|
||||
}
|
||||
case GGML_Q8_0: {
|
||||
float d = fp16_to_f32(*(const uint16_t *)p);
|
||||
const int8_t *q = (const int8_t *)(p + 2);
|
||||
for (i = 0; i < 32; i++) out[i] = (float)q[i] * d;
|
||||
break;
|
||||
}
|
||||
case GGML_Q4_0: {
|
||||
float d = fp16_to_f32(*(const uint16_t *)p);
|
||||
const unsigned char *q = p + 2;
|
||||
for (i = 0; i < 16; i++) {
|
||||
int lo = q[i] & 0x0F;
|
||||
int hi = (q[i] >> 4) & 0x0F;
|
||||
out[i * 2 + 0] = (float)(lo - 8) * d;
|
||||
out[i * 2 + 1] = (float)(hi - 8) * d;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_Q5_0: {
|
||||
float d = fp16_to_f32(*(const uint16_t *)p);
|
||||
const unsigned char *ql = p + 2;
|
||||
const unsigned char *qh = p + 2 + 16; /* 4 字节，bit w 对应权重 w 的高位 */
|
||||
for (int w = 0; w < 32; w++) {
|
||||
int low = (w & 1) ? (ql[w / 2] >> 4) & 0x0F : ql[w / 2] & 0x0F;
|
||||
int high = (qh[w / 8] >> (w % 8)) & 1;
|
||||
out[w] = (float)(low + (high << 4) - 16) * d;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_Q5_1: {
|
||||
float d = fp16_to_f32(*(const uint16_t *)p);
|
||||
float m = fp16_to_f32(*(const uint16_t *)(p + 2));
|
||||
const unsigned char *ql = p + 4;
|
||||
const unsigned char *qh = p + 4 + 16;
|
||||
for (int w = 0; w < 32; w++) {
|
||||
int low = (w & 1) ? (ql[w / 2] >> 4) & 0x0F : ql[w / 2] & 0x0F;
|
||||
int high = (qh[w / 8] >> (w % 8)) & 1;
|
||||
out[w] = (float)(low + (high << 4)) * d + m;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
for (i = 0; i < 32; i++) out[i] = 0.0f;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- 张量 ---- */
|
||||
|
||||
int gguf_has_tensor(const Gguf *g, const char *name) {
|
||||
for (uint64_t i = 0; i < g->tensor_count; i++) {
|
||||
if (strcmp(g->tensor_name[i], name) == 0) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 计算张量元素个数 = prod(dims) */
|
||||
static uint64_t tensor_n(const Gguf *g, uint64_t i) {
|
||||
uint64_t n = 1;
|
||||
for (uint32_t d = 0; d < g->n_dims[i]; d++) n *= g->dims[i][d];
|
||||
return n;
|
||||
}
|
||||
|
||||
long gguf_tensor_to_f32(const Gguf *g, const char *name, float *out) {
|
||||
for (uint64_t i = 0; i < g->tensor_count; i++) {
|
||||
if (strcmp(g->tensor_name[i], name) != 0) continue;
|
||||
uint64_t n = tensor_n(g, i);
|
||||
uint32_t t = g->ggml_type[i];
|
||||
uint32_t bb = type_block_bytes(t);
|
||||
if (bb == 0) return -1;
|
||||
const unsigned char *p = gd(g, g->data_offset + (size_t)g->offset[i]);
|
||||
uint64_t blocks = (n + 31) / 32;
|
||||
uint64_t k = 0;
|
||||
for (uint64_t b = 0; b < blocks; b++) {
|
||||
float tmp[32];
|
||||
dequant_block(t, p + (size_t)b * bb, tmp);
|
||||
for (int j = 0; j < 32 && k < n; j++) out[k++] = tmp[j];
|
||||
}
|
||||
return (long)n;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* ---- 解析 ---- */
|
||||
|
||||
/* 版本自适应长度：GGUF v1 用 u32，v2/v3 用 u64 */
|
||||
static size_t rd_len(int ver, const unsigned char *p, size_t *off) {
|
||||
uint64_t v;
|
||||
if (ver >= 2) { v = rd_u64(p + *off); *off += 8; }
|
||||
else { v = rd_u32(p + *off); *off += 4; }
|
||||
return (size_t)v;
|
||||
}
|
||||
|
||||
/* 跳过 value，返回跳过的字节数。ver 用于数组/字符串长度宽度 */
|
||||
static size_t skip_gval(int ver, const unsigned char *p, uint32_t vt) {
|
||||
switch (vt) {
|
||||
case GGVAL_UINT8: case GGVAL_INT8: case GGVAL_BOOL: return 1;
|
||||
case GGVAL_UINT16: case GGVAL_INT16: return 2;
|
||||
case GGVAL_UINT32: case GGVAL_INT32: case GGVAL_FLOAT32: return 4;
|
||||
case GGVAL_FLOAT64: case GGVAL_UINT64: case GGVAL_INT64: return 8;
|
||||
case GGVAL_STRING: { size_t o = 0; size_t n = rd_len(ver, p, &o); return o + n; }
|
||||
case GGVAL_ARRAY: {
|
||||
size_t o = 0;
|
||||
uint32_t it = rd_u32(p + o); o += 4;
|
||||
size_t n = rd_len(ver, p, &o);
|
||||
const unsigned char *q = p + o;
|
||||
for (size_t i = 0; i < n; i++) q += skip_gval(ver, q, it);
|
||||
return (size_t)(q - p);
|
||||
}
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static char *read_gstr(int ver, const unsigned char *p, size_t *off) {
|
||||
size_t n = rd_len(ver, p, off);
|
||||
char *s = malloc((size_t)n + 1);
|
||||
memcpy(s, p + *off, (size_t)n);
|
||||
s[n] = '\0';
|
||||
*off += (size_t)n;
|
||||
return s;
|
||||
}
|
||||
|
||||
int gguf_open(Gguf *g, const char *path) {
|
||||
memset(g, 0, sizeof(*g));
|
||||
FILE *fh = fopen(path, "rb");
|
||||
if (!fh) return -1;
|
||||
fseek(fh, 0, SEEK_END);
|
||||
long sz = ftell(fh);
|
||||
fseek(fh, 0, SEEK_SET);
|
||||
if (sz <= 0) { fclose(fh); return -2; }
|
||||
unsigned char *buf = malloc((size_t)sz);
|
||||
if (!buf) { fclose(fh); return -3; }
|
||||
if (fread(buf, 1, (size_t)sz, fh) != (size_t)sz) { free(buf); fclose(fh); return -4; }
|
||||
fclose(fh);
|
||||
|
||||
if (sz < 24) { free(buf); return -5; }
|
||||
if (memcmp(buf, "GGUF", 4) != 0) { free(buf); return -6; }
|
||||
g->data = buf;
|
||||
g->size = (size_t)sz;
|
||||
|
||||
size_t off = 4;
|
||||
g->version = rd_u32(buf + off); off += 4;
|
||||
g->tensor_count = rd_len(g->version, buf, &off);
|
||||
g->meta_count = rd_len(g->version, buf, &off);
|
||||
|
||||
/* 探测段顺序：GGUF 生产者可能写“元数据在前”或“张量在前”。
|
||||
张量名总以 .weight 结尾；元数据键不会。 */
|
||||
int meta_first = 1;
|
||||
{
|
||||
size_t o = off;
|
||||
size_t n0 = rd_len(g->version, buf, &o);
|
||||
if (n0 >= 7 && memcmp(buf + o + n0 - 7, ".weight", 7) == 0) meta_first = 0;
|
||||
}
|
||||
|
||||
/* 元数据 */
|
||||
g->meta_key = calloc((size_t)g->meta_count, sizeof(char *));
|
||||
g->meta_type = calloc((size_t)g->meta_count, sizeof(uint32_t));
|
||||
g->meta_off = calloc((size_t)g->meta_count, sizeof(const unsigned char *));
|
||||
g->meta_len = calloc((size_t)g->meta_count, sizeof(size_t));
|
||||
|
||||
/* 张量信息 */
|
||||
g->tensor_name = calloc((size_t)g->tensor_count, sizeof(char *));
|
||||
g->n_dims = calloc((size_t)g->tensor_count, sizeof(uint32_t));
|
||||
g->dims = calloc((size_t)g->tensor_count, sizeof(uint64_t *));
|
||||
g->ggml_type = calloc((size_t)g->tensor_count, sizeof(uint32_t));
|
||||
g->offset = calloc((size_t)g->tensor_count, sizeof(uint64_t));
|
||||
|
||||
#define READ_METADATA() \
|
||||
for (uint64_t i = 0; i < g->meta_count; i++) { \
|
||||
g->meta_key[i] = read_gstr(g->version, buf, &off); \
|
||||
uint32_t vt = rd_u32(buf + off); off += 4; \
|
||||
g->meta_type[i] = vt; \
|
||||
g->meta_off[i] = buf + off; \
|
||||
size_t len = skip_gval(g->version, buf + off, vt); \
|
||||
g->meta_len[i] = len; \
|
||||
off += len; \
|
||||
}
|
||||
#define READ_TENSORS() \
|
||||
for (uint64_t i = 0; i < g->tensor_count; i++) { \
|
||||
g->tensor_name[i] = read_gstr(g->version, buf, &off); \
|
||||
g->n_dims[i] = rd_u32(buf + off); off += 4; \
|
||||
g->dims[i] = malloc((size_t)g->n_dims[i] * sizeof(uint64_t)); \
|
||||
for (uint32_t d = 0; d < g->n_dims[i]; d++) { \
|
||||
g->dims[i][d] = rd_len(g->version, buf, &off); \
|
||||
} \
|
||||
g->ggml_type[i] = rd_u32(buf + off); off += 4; \
|
||||
g->offset[i] = rd_len(g->version, buf, &off); \
|
||||
}
|
||||
|
||||
if (meta_first) { READ_METADATA(); READ_TENSORS(); }
|
||||
else { READ_TENSORS(); READ_METADATA(); }
|
||||
|
||||
#undef READ_METADATA
|
||||
#undef READ_TENSORS
|
||||
|
||||
g->data_offset = off;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 查找元数据索引，未命中返回 -1 */
|
||||
static ptrdiff_t meta_index(const Gguf *g, const char *key) {
|
||||
for (uint64_t i = 0; i < g->meta_count; i++) {
|
||||
if (strcmp(g->meta_key[i], key) == 0) return (ptrdiff_t)i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int gguf_meta_u32(const Gguf *g, const char *key, uint32_t *v) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
switch (g->meta_type[i]) {
|
||||
case GGVAL_UINT8: *v = p[0]; return 1;
|
||||
case GGVAL_UINT16: *v = rd_u16(p); return 1;
|
||||
case GGVAL_UINT32: *v = rd_u32(p); return 1;
|
||||
case GGVAL_INT32: *v = rd_u32(p); return 1;
|
||||
case GGVAL_UINT64: *v = (uint32_t)rd_u64(p); return 1;
|
||||
case GGVAL_INT64: *v = (uint32_t)rd_u64(p); return 1;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int gguf_meta_f32(const Gguf *g, const char *key, float *v) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
if (g->meta_type[i] == GGVAL_FLOAT32) { *v = rd_f32(p); return 1; }
|
||||
if (g->meta_type[i] == GGVAL_FLOAT64) {
|
||||
union { double d; uint64_t b; } u; memcpy(&u.b, p, 8); *v = (float)u.d; return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int gguf_meta_u32_array(const Gguf *g, const char *key, uint32_t *out, uint64_t max) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
if (g->meta_type[i] != GGVAL_ARRAY) return 0;
|
||||
uint32_t it = rd_u32(p);
|
||||
size_t o = 4;
|
||||
size_t n = rd_len(g->version, p, &o);
|
||||
if (it != GGVAL_UINT32) return 0;
|
||||
size_t take = n < max ? n : max;
|
||||
for (size_t k = 0; k < take; k++) out[k] = rd_u32(p + o + k * 4);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int gguf_meta_u8_array(const Gguf *g, const char *key, uint8_t *out, uint64_t max) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
if (g->meta_type[i] == GGVAL_ARRAY) {
|
||||
uint32_t it = rd_u32(p);
|
||||
size_t o = 4;
|
||||
size_t n = rd_len(g->version, p, &o);
|
||||
if (it != GGVAL_UINT8) return 0;
|
||||
size_t take = n < max ? n : max;
|
||||
for (size_t k = 0; k < take; k++) out[k] = p[o + k];
|
||||
return 1;
|
||||
}
|
||||
if (g->meta_type[i] == GGVAL_STRING) {
|
||||
size_t o = 0;
|
||||
size_t n = rd_len(g->version, p, &o);
|
||||
size_t take = n < max ? n : max;
|
||||
for (size_t k = 0; k < take; k++) out[k] = p[o + k];
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int gguf_meta_string(const Gguf *g, const char *key, char *out, size_t max) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
if (g->meta_type[i] == GGVAL_STRING) {
|
||||
size_t o = 0;
|
||||
size_t n = rd_len(g->version, p, &o);
|
||||
size_t take = (size_t)n < max ? (size_t)n : max - 1;
|
||||
memcpy(out, p + o, take);
|
||||
out[take] = '\0';
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int gguf_meta_string_array(const Gguf *g, const char *key, char ***out, int *count) {
|
||||
ptrdiff_t i = meta_index(g, key);
|
||||
if (i < 0) return 0;
|
||||
const unsigned char *p = g->meta_off[i];
|
||||
if (g->meta_type[i] != GGVAL_ARRAY) return 0;
|
||||
uint32_t it = rd_u32(p);
|
||||
size_t o = 4;
|
||||
size_t n = rd_len(g->version, p, &o);
|
||||
if (it != GGVAL_STRING) return 0;
|
||||
char **arr = calloc((size_t)n, sizeof(char *));
|
||||
const unsigned char *q = p + o;
|
||||
for (size_t k = 0; k < n; k++) {
|
||||
size_t so = 0;
|
||||
size_t sl = rd_len(g->version, q, &so);
|
||||
arr[k] = malloc((size_t)sl + 1);
|
||||
memcpy(arr[k], q + so, (size_t)sl);
|
||||
arr[k][sl] = '\0';
|
||||
q += so + (size_t)sl;
|
||||
}
|
||||
if (out) *out = arr;
|
||||
if (count) *count = (int)n;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void gguf_free_string_array(char **arr, int count) {
|
||||
if (!arr) return;
|
||||
for (int i = 0; i < count; i++) free(arr[i]);
|
||||
free(arr);
|
||||
}
|
||||
|
||||
void gguf_close(Gguf *g) {
|
||||
if (!g) return;
|
||||
for (uint64_t i = 0; i < g->tensor_count; i++) {
|
||||
free(g->tensor_name[i]);
|
||||
free(g->dims[i]);
|
||||
}
|
||||
for (uint64_t i = 0; i < g->meta_count; i++) free(g->meta_key[i]);
|
||||
free(g->tensor_name);
|
||||
free(g->n_dims);
|
||||
free(g->dims);
|
||||
free(g->ggml_type);
|
||||
free(g->offset);
|
||||
free(g->meta_key);
|
||||
free(g->meta_type);
|
||||
free(g->meta_off);
|
||||
free(g->meta_len);
|
||||
free((void *)g->data);
|
||||
memset(g, 0, sizeof(*g));
|
||||
}
|
||||
|
||||
static const char *type_name(uint32_t t) {
|
||||
switch (t) {
|
||||
case GGML_F32: return "F32";
|
||||
case GGML_F16: return "F16";
|
||||
case GGML_BF16: return "BF16";
|
||||
case GGML_Q8_0: return "Q8_0";
|
||||
case GGML_Q4_0: return "Q4_0";
|
||||
case GGML_Q5_0: return "Q5_0";
|
||||
case GGML_Q5_1: return "Q5_1";
|
||||
default: return "?";
|
||||
}
|
||||
}
|
||||
|
||||
void gguf_inspect(const Gguf *g, const char *path, FILE *out) {
|
||||
fprintf(out, "==== GGUF 文件: %s ====\n", path);
|
||||
fprintf(out, "version=%u tensors=%llu metadata=%llu data_offset=%zu\n",
|
||||
g->version,
|
||||
(unsigned long long)g->tensor_count,
|
||||
(unsigned long long)g->meta_count,
|
||||
g->data_offset);
|
||||
fprintf(out, "--- 张量表 ---\n");
|
||||
for (uint64_t i = 0; i < g->tensor_count; i++) {
|
||||
uint64_t n = tensor_n(g, i);
|
||||
fprintf(out, " %-32s dims=", g->tensor_name[i]);
|
||||
for (uint32_t d = 0; d < g->n_dims[i]; d++) fprintf(out, "%llu%s",
|
||||
(unsigned long long)g->dims[i][d], d + 1 < g->n_dims[i] ? "x" : "");
|
||||
fprintf(out, " type=%s elems=%llu bytes~%llu\n",
|
||||
type_name(g->ggml_type[i]), (unsigned long long)n,
|
||||
(unsigned long long)(type_block_bytes(g->ggml_type[i]) * ((n + 31) / 32)));
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
#ifndef MO_GGUF_H
|
||||
#define MO_GGUF_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* GGUF (llama.cpp) 容器读取。v2/v3。 */
|
||||
typedef struct {
|
||||
const unsigned char *data;
|
||||
size_t size;
|
||||
uint32_t version;
|
||||
uint64_t tensor_count;
|
||||
uint64_t meta_count;
|
||||
size_t data_offset; /* 张量数据段起点（绝对文件偏移） */
|
||||
|
||||
/* 张量表 */
|
||||
char **tensor_name;
|
||||
uint32_t *n_dims;
|
||||
uint64_t **dims; /* [i][d] */
|
||||
uint32_t *ggml_type;
|
||||
uint64_t *offset; /* 相对 data_offset */
|
||||
|
||||
/* 元数据（供查询） */
|
||||
char **meta_key;
|
||||
uint32_t *meta_type;
|
||||
const unsigned char **meta_off; /* value 起始（在 buffer 内） */
|
||||
size_t *meta_len; /* value 字节数 */
|
||||
} Gguf;
|
||||
|
||||
/* 打开并解析文件。成功返回 0，失败返回负。 */
|
||||
int gguf_open(Gguf *g, const char *path);
|
||||
void gguf_close(Gguf *g);
|
||||
|
||||
/* 元数据查询：返回 1 命中，0 未命中 */
|
||||
int gguf_meta_u32(const Gguf *g, const char *key, uint32_t *v);
|
||||
int gguf_meta_f32(const Gguf *g, const char *key, float *v);
|
||||
int gguf_meta_u32_array(const Gguf *g, const char *key, uint32_t *out, uint64_t max);
|
||||
int gguf_meta_u8_array(const Gguf *g, const char *key, uint8_t *out, uint64_t max);
|
||||
int gguf_meta_string(const Gguf *g, const char *key, char *out, size_t max);
|
||||
/* 读字符串数组（如 tokenizer.ggml.tokens/merges）。成功返回 1，*out 需调用 gguf_free_string_array 释放 */
|
||||
int gguf_meta_string_array(const Gguf *g, const char *key, char ***out, int *count);
|
||||
void gguf_free_string_array(char **arr, int count);
|
||||
|
||||
/* 打印元数据与张量表到 out（任意 GGUF 都能看）。 */
|
||||
void gguf_inspect(const Gguf *g, const char *path, FILE *out);
|
||||
|
||||
/* 把名为 name 的张量反量化成 f32 写入 out（out 长度需 rows*cols）。成功返回字节数，失败<0 */
|
||||
/* 这里张量按 flat 元素看待，返回元素个数。 */
|
||||
long gguf_tensor_to_f32(const Gguf *g, const char *name, float *out);
|
||||
int gguf_has_tensor(const Gguf *g, const char *name);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_GGUF_H */
|
||||
+442
@@ -0,0 +1,442 @@
|
||||
#include "model/llama.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "infer/sample.h"
|
||||
#include "model/gguf.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
/* ---- 微型算子 ---- */
|
||||
static float siluf(float x) { return x / (1.0f + expf(-x)); }
|
||||
|
||||
static void rmsnorm(const float *x, const float *w, float eps, int d, float *y) {
|
||||
float sq = 0.0f;
|
||||
for (int i = 0; i < d; i++) sq += x[i] * x[i];
|
||||
float inv = 1.0f / sqrtf(sq / (float)d + eps);
|
||||
for (int i = 0; i < d; i++) y[i] = x[i] * inv * w[i];
|
||||
}
|
||||
|
||||
/* y[cols] = x[k] @ W[k,cols] （行主序 W=(k,cols)） */
|
||||
static void matvec(const float *x, const float *W, int k, int cols, float *y) {
|
||||
for (int j = 0; j < cols; j++) {
|
||||
float s = 0.0f;
|
||||
for (int i = 0; i < k; i++) s += x[i] * W[i * cols + j];
|
||||
y[j] = s;
|
||||
}
|
||||
}
|
||||
|
||||
static void silu_inplace(float *x, int n) { for (int i = 0; i < n; i++) x[i] = siluf(x[i]); }
|
||||
|
||||
/* RoPE (NeoX half-rotate) 作用于一个 head 向量（长度 dim），位置 pos */
|
||||
static void rope_inplace(const LlamaModel *m, float *v, int pos) {
|
||||
int half = m->c.head_dim / 2;
|
||||
const float *c = m->cos_cache[pos];
|
||||
const float *s = m->sin_cache[pos];
|
||||
for (int i = 0; i < half; i++) {
|
||||
float a = v[i], b = v[i + half];
|
||||
v[i] = a * c[i] - b * s[i];
|
||||
v[i + half] = b * c[i] + a * s[i];
|
||||
}
|
||||
}
|
||||
|
||||
void llama_embed(const LlamaModel *m, int token, float *x) {
|
||||
const float *te = m->token_embd + (size_t)token * m->c.n_embd;
|
||||
memcpy(x, te, (size_t)m->c.n_embd * sizeof(float));
|
||||
}
|
||||
|
||||
/* ---- MoE：router softmax → top-k → 重归一化 → SwiGLU 专家加权和 ---- */
|
||||
static void moe_ffn(const LlamaModel *m, int l, const float *x, float *out) {
|
||||
int d = m->c.n_embd, ne = m->c.n_expert, used = m->c.n_expert_used, de = m->c.ffn_dim;
|
||||
float *probs = malloc((size_t)ne * sizeof(float));
|
||||
matvec(x, m->router[l], d, ne, probs);
|
||||
float mx = probs[0];
|
||||
for (int e = 1; e < ne; e++) if (probs[e] > mx) mx = probs[e];
|
||||
float sum = 0.0f;
|
||||
for (int e = 0; e < ne; e++) { probs[e] = expf(probs[e] - mx); sum += probs[e]; }
|
||||
for (int e = 0; e < ne; e++) probs[e] /= sum;
|
||||
|
||||
int *idx = malloc((size_t)used * sizeof(int));
|
||||
for (int t = 0; t < used; t++) {
|
||||
int best = -1; float bp = -1.0f;
|
||||
for (int e = 0; e < ne; e++) {
|
||||
int taken = 0;
|
||||
for (int q = 0; q < t; q++) if (idx[q] == e) { taken = 1; break; }
|
||||
if (!taken && probs[e] > bp) { bp = probs[e]; best = e; }
|
||||
}
|
||||
idx[t] = best;
|
||||
}
|
||||
float wsum = 0.0f;
|
||||
for (int t = 0; t < used; t++) wsum += probs[idx[t]];
|
||||
|
||||
float *g = malloc((size_t)de * sizeof(float));
|
||||
float *up = malloc((size_t)de * sizeof(float));
|
||||
float *eo = malloc((size_t)d * sizeof(float));
|
||||
for (int i = 0; i < d; i++) out[i] = 0.0f;
|
||||
for (int t = 0; t < used; t++) {
|
||||
int e = idx[t];
|
||||
float w = probs[e] / (wsum + 1e-9f);
|
||||
matvec(x, m->ffn_gate[l][e], d, de, g); silu_inplace(g, de);
|
||||
matvec(x, m->ffn_up[l][e], d, de, up);
|
||||
for (int i = 0; i < de; i++) g[i] *= up[i];
|
||||
matvec(g, m->ffn_down[l][e], de, d, eo);
|
||||
for (int i = 0; i < d; i++) out[i] += w * eo[i];
|
||||
}
|
||||
free(probs); free(idx); free(g); free(up); free(eo);
|
||||
}
|
||||
|
||||
/* ---- 稠密 SwiGLU：down(silu(gate(x)) * up(x)) ---- */
|
||||
static void dense_ffn(const LlamaModel *m, int l, const float *x, float *out) {
|
||||
int d = m->c.n_embd, de = m->c.ffn_dim;
|
||||
float *g = malloc((size_t)de * sizeof(float));
|
||||
float *up = malloc((size_t)de * sizeof(float));
|
||||
matvec(x, m->df_gate[l], d, de, g); silu_inplace(g, de);
|
||||
matvec(x, m->df_up[l], d, de, up);
|
||||
for (int i = 0; i < de; i++) g[i] *= up[i];
|
||||
matvec(g, m->df_down[l], de, d, out);
|
||||
free(g); free(up);
|
||||
}
|
||||
|
||||
/* ---- 单位置前向 ---- */
|
||||
void llama_forward(LlamaModel *m, int pos, const float *x, float *logits) {
|
||||
const LlamaConfig *c = &m->c;
|
||||
int d = c->n_embd, H = c->n_head, HK = c->n_head_kv, hd = c->head_dim;
|
||||
int hpk = H / HK; /* 每组 kv 对应查询头数 */
|
||||
int L = c->n_layer, V = c->vocab;
|
||||
|
||||
float *a = malloc((size_t)d * sizeof(float));
|
||||
float *q = malloc((size_t)(H * hd) * sizeof(float));
|
||||
float *k = malloc((size_t)(HK * hd) * sizeof(float));
|
||||
float *v = malloc((size_t)(HK * hd) * sizeof(float));
|
||||
float *att= malloc((size_t)(H * hd) * sizeof(float));
|
||||
float *o = malloc((size_t)d * sizeof(float));
|
||||
float *ffn= malloc((size_t)d * sizeof(float));
|
||||
float *scores = malloc((size_t)(m->c.max_seq) * sizeof(float));
|
||||
|
||||
float *h = malloc((size_t)d * sizeof(float));
|
||||
memcpy(h, x, (size_t)d * sizeof(float));
|
||||
float isq = 1.0f / sqrtf((float)hd);
|
||||
|
||||
for (int l = 0; l < L; l++) {
|
||||
rmsnorm(h, m->attn_norm[l], c->rmsnorm_eps, d, a);
|
||||
matvec(a, m->attn_q[l], d, H * hd, q);
|
||||
matvec(a, m->attn_k[l], d, HK * hd, k);
|
||||
matvec(a, m->attn_v[l], d, HK * hd, v);
|
||||
|
||||
for (int hh = 0; hh < H; hh++) rope_inplace(m, q + (size_t)hh * hd, pos);
|
||||
for (int hh = 0; hh < HK; hh++) rope_inplace(m, k + (size_t)hh * hd, pos);
|
||||
|
||||
/* 存 KV cache */
|
||||
memcpy(m->k_cache[l] + (size_t)pos * HK * hd, k, (size_t)HK * hd * sizeof(float));
|
||||
memcpy(m->v_cache[l] + (size_t)pos * HK * hd, v, (size_t)HK * hd * sizeof(float));
|
||||
|
||||
for (int hh = 0; hh < H; hh++) {
|
||||
int kvh = hh / hpk;
|
||||
const float *qh = q + (size_t)hh * hd;
|
||||
for (int j = 0; j <= pos; j++) {
|
||||
const float *kj = m->k_cache[l] + ((size_t)j * HK + kvh) * hd;
|
||||
float s = 0.0f;
|
||||
for (int t = 0; t < hd; t++) s += qh[t] * kj[t];
|
||||
scores[j] = s * isq;
|
||||
}
|
||||
float mxs = scores[0];
|
||||
for (int j = 1; j <= pos; j++) if (scores[j] > mxs) mxs = scores[j];
|
||||
float ss = 0.0f;
|
||||
for (int j = 0; j <= pos; j++) { scores[j] = expf(scores[j] - mxs); ss += scores[j]; }
|
||||
float is = 1.0f / ss;
|
||||
for (int j = 0; j <= pos; j++) scores[j] *= is;
|
||||
for (int t = 0; t < hd; t++) {
|
||||
float acc = 0.0f;
|
||||
for (int j = 0; j <= pos; j++) {
|
||||
const float *vj = m->v_cache[l] + ((size_t)j * HK + kvh) * hd;
|
||||
acc += scores[j] * vj[t];
|
||||
}
|
||||
att[(size_t)hh * hd + t] = acc;
|
||||
}
|
||||
}
|
||||
matvec(att, m->attn_o[l], d, d, o);
|
||||
for (int i = 0; i < d; i++) h[i] += o[i];
|
||||
|
||||
rmsnorm(h, m->ffn_norm[l], c->rmsnorm_eps, d, a);
|
||||
if (c->is_moe) moe_ffn(m, l, a, ffn); else dense_ffn(m, l, a, ffn);
|
||||
for (int i = 0; i < d; i++) h[i] += ffn[i];
|
||||
}
|
||||
|
||||
rmsnorm(h, m->output_norm, c->rmsnorm_eps, d, a);
|
||||
matvec(a, m->output, d, V, logits);
|
||||
|
||||
free(a); free(q); free(k); free(v); free(att); free(o); free(ffn); free(scores); free(h);
|
||||
}
|
||||
|
||||
/* ---- 生成：自回归，输出 token id 到 out_ids（返回产出 token 数） ---- */
|
||||
int llama_generate(LlamaModel *m, const int *input_ids, int n_input,
|
||||
int max_new, float temperature, int top_k, int *out_ids) {
|
||||
const int V = m->c.vocab, maxseq = m->c.max_seq;
|
||||
float *logits = malloc((size_t)V * sizeof(float));
|
||||
float *x = malloc((size_t)m->c.n_embd * sizeof(float));
|
||||
int *ids = malloc((size_t)(maxseq + 8) * sizeof(int));
|
||||
|
||||
int n = n_input < maxseq ? n_input : maxseq;
|
||||
for (int i = 0; i < n; i++) ids[i] = input_ids[i];
|
||||
for (int i = 0; i < n; i++) { llama_embed(m, ids[i], x); llama_forward(m, i, x, logits); }
|
||||
|
||||
int pos = n, produced = 0;
|
||||
for (int t = 0; t < max_new; t++) {
|
||||
if (pos >= maxseq) break;
|
||||
int next = sample_top_k(logits, V, temperature, top_k);
|
||||
if (out_ids) out_ids[produced] = next;
|
||||
llama_embed(m, next, x);
|
||||
llama_forward(m, pos, x, logits);
|
||||
pos++;
|
||||
produced++;
|
||||
}
|
||||
free(logits); free(x); free(ids);
|
||||
return produced;
|
||||
}
|
||||
|
||||
/* ---- 名字精确匹配 ---- */
|
||||
static int parse_blk(const char *name, int *i, const char **suffix) {
|
||||
if (strncmp(name, "blk.", 4) != 0) return 0;
|
||||
const char *p = name + 4;
|
||||
if (*p < '0' || *p > '9') return 0;
|
||||
*i = 0;
|
||||
while (*p >= '0' && *p <= '9') { *i = *i * 10 + (*p - '0'); p++; }
|
||||
if (*p != '.') return 0;
|
||||
*suffix = p;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int parse_expert(const char *name, int *i, int *e, const char **suffix) {
|
||||
static const char *pfx = ".ffn_experts.";
|
||||
const char *s;
|
||||
if (!parse_blk(name, i, &s)) return 0;
|
||||
if (strncmp(s, pfx, strlen(pfx)) != 0) return 0;
|
||||
const char *q = s + strlen(pfx);
|
||||
if (*q < '0' || *q > '9') return 0;
|
||||
*e = 0;
|
||||
while (*q >= '0' && *q <= '9') { *e = *e * 10 + (*q - '0'); q++; }
|
||||
if (*q != '.') return 0;
|
||||
*suffix = q;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* 装载一个名为 name 的张量 buf（长度 n，浮点已反量化） */
|
||||
static void load_w(LlamaModel *m, const char *name, const float *buf, uint64_t n) {
|
||||
LlamaConfig *c = &m->c;
|
||||
(void)n;
|
||||
if (strcmp(name, "token_embd.weight") == 0) { memcpy(m->token_embd, buf, (size_t)c->vocab * c->n_embd * 4); return; }
|
||||
if (strcmp(name, "output_norm.weight") == 0) { memcpy(m->output_norm, buf, (size_t)c->n_embd * 4); return; }
|
||||
if (strcmp(name, "output.weight") == 0) { memcpy(m->output, buf, (size_t)c->n_embd * c->vocab * 4); return; }
|
||||
|
||||
int i, e;
|
||||
const char *s;
|
||||
if (parse_expert(name, &i, &e, &s)) {
|
||||
if (strcmp(s, ".ffn_gate.weight") == 0) memcpy(m->ffn_gate[i][e], buf, (size_t)c->n_embd * c->ffn_dim * 4);
|
||||
else if (strcmp(s, ".ffn_up.weight") == 0) memcpy(m->ffn_up[i][e], buf, (size_t)c->n_embd * c->ffn_dim * 4);
|
||||
else if (strcmp(s, ".ffn_down.weight") == 0) memcpy(m->ffn_down[i][e], buf, (size_t)c->ffn_dim * c->n_embd * 4);
|
||||
return;
|
||||
}
|
||||
if (parse_blk(name, &i, &s)) {
|
||||
if (strcmp(s, ".attn_norm.weight") == 0) memcpy(m->attn_norm[i], buf, (size_t)c->n_embd * 4);
|
||||
else if (strcmp(s, ".attn_q.weight") == 0) memcpy(m->attn_q[i], buf, (size_t)c->n_embd * c->n_head * c->head_dim * 4);
|
||||
else if (strcmp(s, ".attn_k.weight") == 0) memcpy(m->attn_k[i], buf, (size_t)c->n_embd * c->n_head_kv * c->head_dim * 4);
|
||||
else if (strcmp(s, ".attn_v.weight") == 0) memcpy(m->attn_v[i], buf, (size_t)c->n_embd * c->n_head_kv * c->head_dim * 4);
|
||||
else if (strcmp(s, ".attn_o.weight") == 0) memcpy(m->attn_o[i], buf, (size_t)c->n_embd * c->n_embd * 4);
|
||||
else if (strcmp(s, ".ffn_norm.weight") == 0) memcpy(m->ffn_norm[i], buf, (size_t)c->n_embd * 4);
|
||||
else if (strcmp(s, ".ffn_gate_inp.weight") == 0) memcpy(m->router[i], buf, (size_t)c->n_embd * c->n_expert * 4);
|
||||
else if (strcmp(s, ".ffn_gate.weight") == 0) memcpy(m->df_gate[i], buf, (size_t)c->n_embd * c->ffn_dim * 4);
|
||||
else if (strcmp(s, ".ffn_up.weight") == 0) memcpy(m->df_up[i], buf, (size_t)c->n_embd * c->ffn_dim * 4);
|
||||
else if (strcmp(s, ".ffn_down.weight") == 0) memcpy(m->df_down[i], buf, (size_t)c->ffn_dim * c->n_embd * 4);
|
||||
return;
|
||||
}
|
||||
fprintf(stderr, "[warn] 忽略未知权重: %s\n", name);
|
||||
}
|
||||
|
||||
/* ---- 加载 GGUF ---- */
|
||||
static int get_u32(const Gguf *g, const char *k, int def, int *out) {
|
||||
uint32_t v; if (gguf_meta_u32(g, k, &v)) { *out = (int)v; return 1; } *out = def; return 0;
|
||||
}
|
||||
static int get_f32(const Gguf *g, const char *k, float def, float *out) {
|
||||
if (gguf_meta_f32(g, k, out)) { return 1; }
|
||||
*out = def;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int llama_load(LlamaModel *m, const char *path) {
|
||||
memset(m, 0, sizeof(*m));
|
||||
Gguf g;
|
||||
if (gguf_open(&g, path) != 0) { fprintf(stderr, "[err] GGUF 打开失败: %s\n", path); return -1; }
|
||||
LlamaConfig *c = &m->c;
|
||||
get_u32(&g, "llama.block_count", 4, &c->n_layer);
|
||||
get_u32(&g, "llama.embedding_length", 512, &c->n_embd);
|
||||
get_u32(&g, "llama.attention.head_count", 8, &c->n_head);
|
||||
get_u32(&g, "llama.attention.head_count_kv", c->n_head, &c->n_head_kv);
|
||||
get_f32(&g, "llama.attention.layer_norm_rms_epsilon", 1e-5f, &c->rmsnorm_eps);
|
||||
get_u32(&g, "llama.feed_forward_length", 1024, &c->ffn_dim);
|
||||
get_u32(&g, "llama.expert_count", 0, &c->n_expert);
|
||||
get_u32(&g, "llama.expert_used_count", 2, &c->n_expert_used);
|
||||
get_u32(&g, "llama.context_length", 256, &c->max_seq);
|
||||
get_u32(&g, "llama.rope.dimension_count", 0, &c->head_dim);
|
||||
get_f32(&g, "llama.rope.freq_base", 10000.0f, &c->rope_theta);
|
||||
get_u32(&g, "llama.vocab_size", 0, &c->vocab);
|
||||
c->is_moe = c->n_expert > 0;
|
||||
if (c->head_dim == 0) c->head_dim = c->n_embd / c->n_head;
|
||||
|
||||
/* vocab 从 token_embd 张量推断 */
|
||||
if (c->vocab == 0 && gguf_has_tensor(&g, "token_embd.weight")) {
|
||||
/* 读 dims: ne0 = n_embd, ne1 = vocab */
|
||||
c->vocab = 0;
|
||||
for (uint64_t i = 0; i < g.tensor_count; i++)
|
||||
if (strcmp(g.tensor_name[i], "token_embd.weight") == 0) { c->vocab = (int)g.dims[i][1]; break; }
|
||||
}
|
||||
|
||||
/* 分配权重 */
|
||||
int L = c->n_layer, V = c->vocab, d = c->n_embd, E = c->n_expert, de = c->ffn_dim;
|
||||
m->token_embd = malloc((size_t)V * d * 4);
|
||||
m->output = malloc((size_t)d * V * 4);
|
||||
m->output_norm = malloc((size_t)d * 4);
|
||||
m->attn_norm = calloc(L, sizeof(float *));
|
||||
m->attn_q = calloc(L, sizeof(float *));
|
||||
m->attn_k = calloc(L, sizeof(float *));
|
||||
m->attn_v = calloc(L, sizeof(float *));
|
||||
m->attn_o = calloc(L, sizeof(float *));
|
||||
m->ffn_norm = calloc(L, sizeof(float *));
|
||||
m->router = calloc(L, sizeof(float *));
|
||||
m->ffn_gate = calloc(L, sizeof(float **));
|
||||
m->ffn_up = calloc(L, sizeof(float **));
|
||||
m->ffn_down = calloc(L, sizeof(float **));
|
||||
m->df_gate = calloc(L, sizeof(float *));
|
||||
m->df_up = calloc(L, sizeof(float *));
|
||||
m->df_down = calloc(L, sizeof(float *));
|
||||
m->k_cache = calloc(L, sizeof(float *));
|
||||
m->v_cache = calloc(L, sizeof(float *));
|
||||
size_t kv_per = (size_t)c->max_seq * c->n_head_kv * c->head_dim;
|
||||
|
||||
for (int i = 0; i < L; i++) {
|
||||
m->attn_norm[i] = malloc((size_t)d * 4);
|
||||
m->attn_q[i] = malloc((size_t)d * c->n_head * c->head_dim * 4);
|
||||
m->attn_k[i] = malloc((size_t)d * c->n_head_kv * c->head_dim * 4);
|
||||
m->attn_v[i] = malloc((size_t)d * c->n_head_kv * c->head_dim * 4);
|
||||
m->attn_o[i] = malloc((size_t)d * d * 4);
|
||||
m->ffn_norm[i] = malloc((size_t)d * 4);
|
||||
m->k_cache[i] = malloc(kv_per * 4);
|
||||
m->v_cache[i] = malloc(kv_per * 4);
|
||||
if (c->is_moe) {
|
||||
m->router[i] = malloc((size_t)d * E * 4);
|
||||
m->ffn_gate[i] = calloc((size_t)E, sizeof(float *));
|
||||
m->ffn_up[i] = calloc((size_t)E, sizeof(float *));
|
||||
m->ffn_down[i] = calloc((size_t)E, sizeof(float *));
|
||||
for (int e = 0; e < E; e++) {
|
||||
m->ffn_gate[i][e] = malloc((size_t)d * de * 4);
|
||||
m->ffn_up[i][e] = malloc((size_t)d * de * 4);
|
||||
m->ffn_down[i][e] = malloc((size_t)de * d * 4);
|
||||
}
|
||||
} else {
|
||||
m->df_gate[i] = malloc((size_t)d * de * 4);
|
||||
m->df_up[i] = malloc((size_t)d * de * 4);
|
||||
m->df_down[i] = malloc((size_t)de * d * 4);
|
||||
}
|
||||
}
|
||||
|
||||
/* 位置编码缓存 cos/sin[max_seq][head_dim/2] */
|
||||
int half = c->head_dim / 2;
|
||||
m->cos_cache = malloc((size_t)c->max_seq * sizeof(float *));
|
||||
m->sin_cache = malloc((size_t)c->max_seq * sizeof(float *));
|
||||
for (int p = 0; p < c->max_seq; p++) {
|
||||
m->cos_cache[p] = malloc((size_t)half * 4);
|
||||
m->sin_cache[p] = malloc((size_t)half * 4);
|
||||
for (int i = 0; i < half; i++) {
|
||||
float theta = powf(c->rope_theta, -2.0f * i / (float)c->head_dim);
|
||||
float ang = (float)p * theta;
|
||||
m->cos_cache[p][i] = cosf(ang);
|
||||
m->sin_cache[p][i] = sinf(ang);
|
||||
}
|
||||
}
|
||||
|
||||
/* 逐张量装载 */
|
||||
for (uint64_t i = 0; i < g.tensor_count; i++) {
|
||||
uint64_t n = 1;
|
||||
for (uint32_t dd = 0; dd < g.n_dims[i]; dd++) n *= g.dims[i][dd];
|
||||
float *tmp = malloc((size_t)n * 4);
|
||||
if (gguf_tensor_to_f32(&g, g.tensor_name[i], tmp) > 0)
|
||||
load_w(m, g.tensor_name[i], tmp, n);
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
/* 分词器（tokenizer.ggml.tokens/merges） */
|
||||
{
|
||||
char **toks = NULL; int ntok = 0;
|
||||
if (gguf_meta_string_array(&g, "tokenizer.ggml.tokens", &toks, &ntok) && ntok > 0) {
|
||||
char **mrg = NULL; int nmrg = 0;
|
||||
gguf_meta_string_array(&g, "tokenizer.ggml.merges", &mrg, &nmrg);
|
||||
llmtok_init(&m->tok, toks, ntok, mrg, nmrg);
|
||||
gguf_free_string_array(toks, ntok);
|
||||
gguf_free_string_array(mrg, nmrg);
|
||||
}
|
||||
}
|
||||
|
||||
gguf_close(&g);
|
||||
m->is_loaded = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int llama_tokenize(const LlamaModel *m, const char *text, int *ids, int max) {
|
||||
if (!m->is_loaded || m->tok.vocab_size == 0) return -1;
|
||||
return llmtok_encode(&m->tok, text, ids, max);
|
||||
}
|
||||
|
||||
int llama_detokenize(const LlamaModel *m, const int *ids, int n, char *out, int max) {
|
||||
if (!m->is_loaded || m->tok.vocab_size == 0) return -1;
|
||||
return llmtok_decode(&m->tok, ids, n, out, max);
|
||||
}
|
||||
|
||||
void llama_free(LlamaModel *m) {
|
||||
if (!m) return;
|
||||
int L = m->c.n_layer, E = m->c.n_expert;
|
||||
llmtok_free(&m->tok);
|
||||
free(m->token_embd); free(m->output); free(m->output_norm);
|
||||
for (int i = 0; i < L; i++) {
|
||||
free(m->attn_norm[i]); free(m->attn_q[i]); free(m->attn_k[i]);
|
||||
free(m->attn_v[i]); free(m->attn_o[i]); free(m->ffn_norm[i]);
|
||||
free(m->k_cache[i]); free(m->v_cache[i]);
|
||||
free(m->router[i]);
|
||||
if (m->ffn_gate[i]) {
|
||||
for (int e = 0; e < E; e++) { free(m->ffn_gate[i][e]); free(m->ffn_up[i][e]); free(m->ffn_down[i][e]); }
|
||||
}
|
||||
free(m->ffn_gate[i]); free(m->ffn_up[i]); free(m->ffn_down[i]);
|
||||
free(m->df_gate[i]); free(m->df_up[i]); free(m->df_down[i]);
|
||||
}
|
||||
free(m->attn_norm); free(m->attn_q); free(m->attn_k); free(m->attn_v);
|
||||
free(m->attn_o); free(m->ffn_norm); free(m->router);
|
||||
free(m->ffn_gate); free(m->ffn_up); free(m->ffn_down);
|
||||
free(m->df_gate); free(m->df_up); free(m->df_down);
|
||||
free(m->k_cache); free(m->v_cache);
|
||||
for (int p = 0; p < m->c.max_seq; p++) { free(m->cos_cache[p]); free(m->sin_cache[p]); }
|
||||
free(m->cos_cache); free(m->sin_cache);
|
||||
memset(m, 0, sizeof(*m));
|
||||
}
|
||||
|
||||
int64_t llama_param_count(const LlamaModel *m) {
|
||||
const LlamaConfig *c = &m->c;
|
||||
int64_t n = 0;
|
||||
n += (int64_t)c->vocab * c->n_embd; /* token_embd */
|
||||
n += (int64_t)c->n_embd * c->vocab; /* output */
|
||||
n += c->n_embd; /* output_norm */
|
||||
for (int i = 0; i < c->n_layer; i++) {
|
||||
n += c->n_embd; /* attn_norm */
|
||||
n += (int64_t)c->n_embd * (c->n_head * c->head_dim); /* q */
|
||||
n += (int64_t)c->n_embd * (c->n_head_kv * c->head_dim); /* k,v */
|
||||
n += (int64_t)c->n_embd * (c->n_head_kv * c->head_dim);
|
||||
n += (int64_t)c->n_embd * c->n_embd; /* o */
|
||||
n += c->n_embd; /* ffn_norm */
|
||||
if (c->is_moe) {
|
||||
n += (int64_t)c->n_embd * c->n_expert; /* router */
|
||||
n += 3LL * c->n_expert * c->n_embd * c->ffn_dim;
|
||||
} else {
|
||||
n += 3LL * c->n_embd * c->ffn_dim;
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#ifndef MO_LLAMA_H
|
||||
#define MO_LLAMA_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "model/llama_tokenizer.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* llama / Mixtral 架构。所有权重以 (in, out) 行主序存储，C 端直接 x @ W。 */
|
||||
typedef struct {
|
||||
int vocab, n_layer, n_embd, n_head, n_head_kv, head_dim, ffn_dim;
|
||||
int n_expert, n_expert_used, max_seq;
|
||||
float rmsnorm_eps, rope_theta;
|
||||
int is_moe; /* 1=Mixrtal MoE, 0=dense SwiGLU */
|
||||
} LlamaConfig;
|
||||
|
||||
typedef struct {
|
||||
LlamaConfig c;
|
||||
|
||||
float *token_embd; /* vocab * n_embd */
|
||||
float *output_norm; /* n_embd */
|
||||
float *output; /* n_embd * vocab */
|
||||
|
||||
float **attn_norm, **attn_q, **attn_k, **attn_v, **attn_o, **ffn_norm; /* [L] */
|
||||
float **router; /* [L] n_embd * n_expert */
|
||||
float ***ffn_gate, ***ffn_up, ***ffn_down; /* [L][E] */
|
||||
float **df_gate, **df_up, **df_down; /* [L] 稠密路径 */
|
||||
|
||||
float **cos_cache, **sin_cache; /* [max_seq][head_dim/2] */
|
||||
|
||||
/* KV cache（内部使用） */
|
||||
float **k_cache, **v_cache; /* [L] max_seq * n_head_kv * head_dim */
|
||||
|
||||
LlamaTokenizer tok; /* 分词器（若 GGUF 带 tokenizer.ggml.tokens） */
|
||||
|
||||
int is_loaded;
|
||||
} LlamaModel;
|
||||
|
||||
int llama_load(LlamaModel *m, const char *path);
|
||||
void llama_free(LlamaModel *m);
|
||||
int64_t llama_param_count(const LlamaModel *m);
|
||||
|
||||
/* 文本 <-> token id（若模型带分词器）。返回 token 数/字节数，负为错 */
|
||||
int llama_tokenize(const LlamaModel *m, const char *text, int *ids, int max);
|
||||
int llama_detokenize(const LlamaModel *m, const int *ids, int n, char *out, int max);
|
||||
|
||||
/* 输入嵌入：x = token_embd[token]（RoPE 在 forward 内处理位置） */
|
||||
void llama_embed(const LlamaModel *m, int token, float *x);
|
||||
|
||||
/* 对位置 pos 前向（会写入内部 KV cache），输出 logits(vocab) */
|
||||
void llama_forward(LlamaModel *m, int pos, const float *x, float *logits);
|
||||
|
||||
/* 自回归生成：从 input_ids 开始，产出 token id 写入 out_ids（长度 max_new），返回产出数。 */
|
||||
int llama_generate(LlamaModel *m, const int *input_ids, int n_input,
|
||||
int max_new, float temperature, int top_k, int *out_ids);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_LLAMA_H */
|
||||
@@ -0,0 +1,168 @@
|
||||
#define _GNU_SOURCE
|
||||
#include "model/llama_tokenizer.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ---- GPT-2 byte_to_unicode 映射 ---- */
|
||||
|
||||
/* b2u_str[b]：byte b 映射成的 unicode 字符的 UTF-8 串；同时生成反向 code2byte[码点]=字节 */
|
||||
static void build_b2u(char b2u[256][5], int code2byte[512]) {
|
||||
int bs[256], nb = 0;
|
||||
for (int i = '!'; i <= '~'; i++) bs[nb++] = i;
|
||||
for (int i = 0xA1; i <= 0xAC; i++) bs[nb++] = i;
|
||||
for (int i = 0xAE; i <= 0xFF; i++) bs[nb++] = i;
|
||||
|
||||
for (int i = 0; i < 512; i++) code2byte[i] = -1;
|
||||
int byte2code[256];
|
||||
for (int b = 0; b < 256; b++) byte2code[b] = -1;
|
||||
for (int i = 0; i < nb; i++) byte2code[bs[i]] = bs[i]; /* 可打印区：字节映射到自身 */
|
||||
int n = 0;
|
||||
for (int b = 0; b < 256; b++) if (byte2code[b] < 0) byte2code[b] = 256 + n++;
|
||||
|
||||
for (int b = 0; b < 256; b++) {
|
||||
int code = byte2code[b];
|
||||
code2byte[code] = b;
|
||||
if (code < 0x80) { b2u[b][0] = (char)code; b2u[b][1] = 0; }
|
||||
else if (code < 0x800) {
|
||||
b2u[b][0] = (char)(0xC0 | (code >> 6));
|
||||
b2u[b][1] = (char)(0x80 | (code & 0x3F));
|
||||
b2u[b][2] = 0;
|
||||
} else {
|
||||
b2u[b][0] = (char)(0xE0 | (code >> 12));
|
||||
b2u[b][1] = (char)(0x80 | ((code >> 6) & 0x3F));
|
||||
b2u[b][2] = (char)(0x80 | (code & 0x3F));
|
||||
b2u[b][3] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* 二分查找 token 字符串 */
|
||||
static int lookup(const LlamaTokenizer *t, const char *s) {
|
||||
int lo = 0, hi = t->vocab_size - 1;
|
||||
while (lo <= hi) {
|
||||
int mid = (lo + hi) / 2;
|
||||
int c = strcmp(t->tokens[t->sort_idx[mid]], s);
|
||||
if (c == 0) return t->sort_idx[mid];
|
||||
if (c < 0) lo = mid + 1; else hi = mid - 1;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int cmp_idx(const void *a, const void *b, void *arg) {
|
||||
char **tokens = (char **)arg;
|
||||
const int *ia = (const int *)a, *ib = (const int *)b;
|
||||
return strcmp(tokens[*ia], tokens[*ib]);
|
||||
}
|
||||
|
||||
int llmtok_init(LlamaTokenizer *t, char **tokens, int vocab_size, char **merges, int num_merges) {
|
||||
memset(t, 0, sizeof(*t));
|
||||
t->vocab_size = vocab_size;
|
||||
t->tokens = calloc((size_t)vocab_size, sizeof(char *));
|
||||
for (int i = 0; i < vocab_size; i++) {
|
||||
size_t n = strlen(tokens[i]);
|
||||
t->tokens[i] = malloc(n + 1);
|
||||
memcpy(t->tokens[i], tokens[i], n + 1);
|
||||
}
|
||||
t->sort_idx = malloc((size_t)vocab_size * sizeof(int));
|
||||
for (int i = 0; i < vocab_size; i++) t->sort_idx[i] = i;
|
||||
qsort_r(t->sort_idx, (size_t)vocab_size, sizeof(int), cmp_idx, t->tokens);
|
||||
|
||||
/* byte -> id */
|
||||
char b2u[256][5];
|
||||
int code2byte[512];
|
||||
build_b2u(b2u, code2byte);
|
||||
t->byte_to_id = malloc(256 * sizeof(int));
|
||||
for (int b = 0; b < 256; b++) t->byte_to_id[b] = lookup(t, b2u[b]);
|
||||
|
||||
/* merges "a b" -> id 对 */
|
||||
t->num_merges = num_merges;
|
||||
t->ma = malloc((size_t)(num_merges > 0 ? num_merges : 1) * sizeof(int));
|
||||
t->mb = malloc((size_t)(num_merges > 0 ? num_merges : 1) * sizeof(int));
|
||||
for (int i = 0; i < num_merges; i++) {
|
||||
const char *s = merges[i];
|
||||
const char *sp = strchr(s, ' ');
|
||||
int la = sp ? (int)(sp - s) : (int)strlen(s);
|
||||
char a[512], b[512];
|
||||
memcpy(a, s, (size_t)la); a[la] = 0;
|
||||
strcpy(b, sp ? sp + 1 : "");
|
||||
t->ma[i] = lookup(t, a);
|
||||
t->mb[i] = lookup(t, b);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void llmtok_free(LlamaTokenizer *t) {
|
||||
if (!t) return;
|
||||
for (int i = 0; i < t->vocab_size; i++) free(t->tokens[i]);
|
||||
free(t->tokens);
|
||||
free(t->byte_to_id);
|
||||
free(t->sort_idx);
|
||||
free(t->ma);
|
||||
free(t->mb);
|
||||
memset(t, 0, sizeof(*t));
|
||||
}
|
||||
|
||||
int llmtok_encode(const LlamaTokenizer *t, const char *text, int *out, int max) {
|
||||
size_t n = strlen(text);
|
||||
if (n == 0) return 0;
|
||||
int *ids = malloc((size_t)(n + 8) * sizeof(int));
|
||||
int len = 0;
|
||||
for (size_t i = 0; i < n; i++) ids[len++] = t->byte_to_id[(unsigned char)text[i]];
|
||||
|
||||
while (1) {
|
||||
int best = -1, bestr = 0x7fffffff;
|
||||
for (int i = 0; i + 1 < len; i++) {
|
||||
int a = ids[i], b = ids[i + 1];
|
||||
for (int r = 0; r < t->num_merges; r++) {
|
||||
if (t->ma[r] == a && t->mb[r] == b && r < bestr) { bestr = r; best = i; break; }
|
||||
}
|
||||
}
|
||||
if (best < 0) break;
|
||||
int a = ids[best], b = ids[best + 1];
|
||||
int la = (int)strlen(t->tokens[a]), lb = (int)strlen(t->tokens[b]);
|
||||
char buf[2048];
|
||||
memcpy(buf, t->tokens[a], (size_t)la);
|
||||
memcpy(buf + la, t->tokens[b], (size_t)lb + 1);
|
||||
int merged = lookup(t, buf);
|
||||
if (merged < 0) break;
|
||||
ids[best] = merged;
|
||||
memmove(&ids[best + 1], &ids[best + 2], (size_t)(len - best - 2) * sizeof(int));
|
||||
len--;
|
||||
}
|
||||
|
||||
if (len > max) { free(ids); return -1; }
|
||||
for (int i = 0; i < len; i++) out[i] = ids[i];
|
||||
free(ids);
|
||||
return len;
|
||||
}
|
||||
|
||||
/* 把一个 UTF-8 字符解码成码点 */
|
||||
static int u8_to_cp(const unsigned char *s, int *len) {
|
||||
unsigned char c = s[0];
|
||||
if (c < 0x80) { *len = 1; return c; }
|
||||
if ((c & 0xE0) == 0xC0) { *len = 2; return ((c & 0x1F) << 6) | (s[1] & 0x3F); }
|
||||
if ((c & 0xF0) == 0xE0) { *len = 3; return ((c & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); }
|
||||
*len = 1; return c;
|
||||
}
|
||||
|
||||
int llmtok_decode(const LlamaTokenizer *t, const int *ids, int n, char *out, int maxcat) {
|
||||
/* 反 byte_to_unicode：码点 -> 字节 */
|
||||
char b2u[256][5];
|
||||
int code2byte[512];
|
||||
build_b2u(b2u, code2byte);
|
||||
int w = 0;
|
||||
for (int i = 0; i < n && w < maxcat; i++) {
|
||||
int id = ids[i];
|
||||
if (id < 0 || id >= t->vocab_size) continue;
|
||||
const char *s = t->tokens[id];
|
||||
while (*s && w < maxcat) {
|
||||
int l; int cp = u8_to_cp((const unsigned char *)s, &l);
|
||||
int b = (cp >= 0 && cp < 512) ? code2byte[cp] : (cp & 0xFF);
|
||||
out[w++] = (char)b;
|
||||
s += l;
|
||||
}
|
||||
}
|
||||
if (w < maxcat) out[w] = '\0'; else out[maxcat - 1] = '\0';
|
||||
return w;
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#ifndef MO_LLAMA_TOKENIZER_H
|
||||
#define MO_LLAMA_TOKENIZER_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* GPT-2 / llama 的 byte-level BPE 分词器（读 GGUF tokenizer.ggml.tokens/merges）。 */
|
||||
typedef struct {
|
||||
int vocab_size;
|
||||
char **tokens; /* 深拷贝，[vocab_size] */
|
||||
int *byte_to_id; /* [256] 字节 -> token id */
|
||||
int *sort_idx; /* [vocab_size] 按 tokens 字符串排序的索引（二分查找用） */
|
||||
int num_merges;
|
||||
int *ma, *mb; /* [num_merges] 被合并的两个 token id */
|
||||
} LlamaTokenizer;
|
||||
|
||||
/* 用 tokens/merges 字符串构建（深拷贝 tokens/merges）。merges 为 "a b" 形式。成功返回 0。 */
|
||||
int llmtok_init(LlamaTokenizer *t, char **tokens, int vocab_size, char **merges, int num_merges);
|
||||
void llmtok_free(LlamaTokenizer *t);
|
||||
|
||||
/* 编码：text -> ids（最多 max 个）。返回 token 数，负为错。 */
|
||||
int llmtok_encode(const LlamaTokenizer *t, const char *text, int *out, int max);
|
||||
/* 解码：ids -> utf-8（最多 max 字节）。返回字节数。 */
|
||||
int llmtok_decode(const LlamaTokenizer *t, const int *ids, int n, char *out, int max);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_LLAMA_TOKENIZER_H */
|
||||
+435
@@ -0,0 +1,435 @@
|
||||
#include "model/model.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "model/gguf.h"
|
||||
|
||||
#define PAP_MAGIC "PAP1"
|
||||
|
||||
static uint32_t rd_u32(const unsigned char *b, size_t *off) {
|
||||
uint32_t v;
|
||||
memcpy(&v, b + *off, 4);
|
||||
*off += 4;
|
||||
return v;
|
||||
}
|
||||
|
||||
static float rd_f32(const unsigned char *b, size_t *off) {
|
||||
float v;
|
||||
memcpy(&v, b + *off, 4);
|
||||
*off += 4;
|
||||
return v;
|
||||
}
|
||||
|
||||
static int expect_magic(const unsigned char *b, size_t *off) {
|
||||
if (memcmp(b + *off, PAP_MAGIC, 4) != 0) return -1;
|
||||
*off += 4;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 精确匹配：sscanf 成功赋值且整个名字都被消费（避免尾部字面量未匹配却部分成功） */
|
||||
static int m1(const char *name, const char *fmt, int *a) {
|
||||
int n = 0;
|
||||
int r = sscanf(name, fmt, a, &n);
|
||||
return (r == 1) && name[n] == '\0';
|
||||
}
|
||||
static int m2(const char *name, const char *fmt, int *a, int *b) {
|
||||
int n = 0;
|
||||
int r = sscanf(name, fmt, a, b, &n);
|
||||
return (r == 2) && name[n] == '\0';
|
||||
}
|
||||
|
||||
/* 把第 layer 层第 expert 个 MoE 专家的 w1/w2 指针（按需分配）填入 m。 */
|
||||
static void ensure_moe_expert(Model *m, int layer, int expert) {
|
||||
if (!m->exp_w1[layer][expert]) {
|
||||
m->exp_w1[layer][expert] = malloc((size_t)m->config.d_model * m->config.d_expert * sizeof(float));
|
||||
m->exp_w2[layer][expert] = malloc((size_t)m->config.d_expert * m->config.d_model * sizeof(float));
|
||||
}
|
||||
}
|
||||
|
||||
static void load_weight(Model *m, const char *name, uint32_t rows, uint32_t cols, const float *src) {
|
||||
(void)rows;
|
||||
(void)cols;
|
||||
ModelConfig *c = &m->config;
|
||||
int l, e;
|
||||
|
||||
if (strcmp(name, "wte") == 0) {
|
||||
memcpy(m->wte, src, (size_t)c->vocab_size * c->d_model * sizeof(float));
|
||||
} else if (strcmp(name, "wpe") == 0) {
|
||||
memcpy(m->wpe, src, (size_t)c->max_seq_len * c->d_model * sizeof(float));
|
||||
} else if (strcmp(name, "norm_final") == 0) {
|
||||
memcpy(m->norm_final, src, (size_t)c->d_model * sizeof(float));
|
||||
} else if (strcmp(name, "lm_head") == 0) {
|
||||
memcpy(m->lm_head, src, (size_t)c->vocab_size * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.norm1%n", &l)) {
|
||||
memcpy(m->l_norm1[l], src, (size_t)c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.attn.wq%n", &l)) {
|
||||
memcpy(m->wq[l], src, (size_t)c->d_model * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.attn.wk%n", &l)) {
|
||||
memcpy(m->wk[l], src, (size_t)c->d_model * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.attn.wv%n", &l)) {
|
||||
memcpy(m->wv[l], src, (size_t)c->d_model * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.attn.wo%n", &l)) {
|
||||
memcpy(m->wo[l], src, (size_t)c->d_model * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.norm2%n", &l)) {
|
||||
memcpy(m->l_norm2[l], src, (size_t)c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.moe.router%n", &l)) {
|
||||
memcpy(m->router[l], src, (size_t)c->d_model * c->moe_n_experts * sizeof(float));
|
||||
} else if (m2(name, "l%d.moe.exp%d.w1%n", &l, &e)) {
|
||||
ensure_moe_expert(m, l, e);
|
||||
memcpy(m->exp_w1[l][e], src, (size_t)c->d_model * c->d_expert * sizeof(float));
|
||||
} else if (m2(name, "l%d.moe.exp%d.w2%n", &l, &e)) {
|
||||
ensure_moe_expert(m, l, e);
|
||||
memcpy(m->exp_w2[l][e], src, (size_t)c->d_expert * c->d_model * sizeof(float));
|
||||
} else if (m1(name, "l%d.ffn.w1%n", &l)) {
|
||||
memcpy(m->ffn_w1[l], src, (size_t)c->d_model * c->d_ff * sizeof(float));
|
||||
} else if (m1(name, "l%d.ffn.w2%n", &l)) {
|
||||
memcpy(m->ffn_w2[l], src, (size_t)c->d_ff * c->d_model * sizeof(float));
|
||||
} else {
|
||||
fprintf(stderr, "[warn] 忽略未知权重: %s\n", name);
|
||||
}
|
||||
}
|
||||
|
||||
/* fp16 -> fp32 */
|
||||
static float half_to_float(uint16_t h) {
|
||||
uint32_t sign = (uint32_t)(h >> 15) & 1u;
|
||||
uint32_t exp = (h >> 10) & 0x1fu;
|
||||
uint32_t mant = h & 0x3ffu;
|
||||
uint32_t bits;
|
||||
float f;
|
||||
if (exp == 0x1fu) { /* inf / nan */
|
||||
bits = (sign << 31) | 0x7f800000u | (mant << 13);
|
||||
memcpy(&f, &bits, 4);
|
||||
return f;
|
||||
}
|
||||
if (exp == 0) { /* 0 或次正规 */
|
||||
if (mant == 0) { bits = sign << 31; memcpy(&f, &bits, 4); return f; }
|
||||
f = (float)mant * 5.9604644775390625e-8f; /* mant * 2^-24 */
|
||||
return sign ? -f : f;
|
||||
}
|
||||
bits = (sign << 31) | ((exp - 15 + 127) << 23) | (mant << 13);
|
||||
memcpy(&f, &bits, 4);
|
||||
return f;
|
||||
}
|
||||
|
||||
/* 把原始权重字节按 dtype 反量化成 f32 写入 dst，返回读走的字节数 */
|
||||
static size_t read_weight_f32(const unsigned char *p, uint32_t rows, uint32_t cols,
|
||||
uint32_t dtype, float *dst) {
|
||||
size_t n = (size_t)rows * cols;
|
||||
if (dtype == 0) {
|
||||
memcpy(dst, p, n * 4);
|
||||
return n * 4;
|
||||
}
|
||||
if (dtype == 1) {
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
uint16_t h;
|
||||
memcpy(&h, p + i * 2, 2);
|
||||
dst[i] = half_to_float(h);
|
||||
}
|
||||
return n * 2;
|
||||
}
|
||||
if (dtype == 2) { /* q8: 每块 [fp16 scale][32 int8] */
|
||||
size_t k = 0, po = 0;
|
||||
size_t nb = (n + 31) / 32;
|
||||
for (size_t b = 0; b < nb; b++) {
|
||||
uint16_t hs;
|
||||
memcpy(&hs, p + po, 2);
|
||||
po += 2;
|
||||
float scale = half_to_float(hs);
|
||||
size_t cnt = (n - k) < 32 ? (n - k) : 32;
|
||||
for (size_t j = 0; j < cnt; j++) {
|
||||
dst[k++] = (float)(int8_t)p[po + j] * (scale / 127.0f);
|
||||
}
|
||||
po += cnt;
|
||||
}
|
||||
return po;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 根据已就绪的 config 分配全部权重存储（供 .pap/.gguf 加载器复用）。失败返回 -1 */
|
||||
static int alloc_weights(Model *m) {
|
||||
ModelConfig *c = &m->config;
|
||||
int L = c->n_layer;
|
||||
m->wte = malloc((size_t)c->vocab_size * c->d_model * sizeof(float));
|
||||
m->wpe = malloc((size_t)c->max_seq_len * c->d_model * sizeof(float));
|
||||
m->l_norm1 = calloc(L, sizeof(float *));
|
||||
m->wq = calloc(L, sizeof(float *));
|
||||
m->wk = calloc(L, sizeof(float *));
|
||||
m->wv = calloc(L, sizeof(float *));
|
||||
m->wo = calloc(L, sizeof(float *));
|
||||
m->l_norm2 = calloc(L, sizeof(float *));
|
||||
m->router = calloc(L, sizeof(float *));
|
||||
m->exp_w1 = calloc(L, sizeof(float **));
|
||||
m->exp_w2 = calloc(L, sizeof(float **));
|
||||
m->ffn_w1 = calloc(L, sizeof(float *));
|
||||
m->ffn_w2 = calloc(L, sizeof(float *));
|
||||
m->norm_final = malloc((size_t)c->d_model * sizeof(float));
|
||||
m->lm_head = malloc((size_t)c->vocab_size * c->d_model * sizeof(float));
|
||||
|
||||
for (int l = 0; l < L; l++) {
|
||||
m->l_norm1[l] = malloc((size_t)c->d_model * sizeof(float));
|
||||
m->wq[l] = malloc((size_t)c->d_model * c->d_model * sizeof(float));
|
||||
m->wk[l] = malloc((size_t)c->d_model * c->d_model * sizeof(float));
|
||||
m->wv[l] = malloc((size_t)c->d_model * c->d_model * sizeof(float));
|
||||
m->wo[l] = malloc((size_t)c->d_model * c->d_model * sizeof(float));
|
||||
m->l_norm2[l] = malloc((size_t)c->d_model * sizeof(float));
|
||||
m->exp_w1[l] = calloc((size_t)c->moe_n_experts, sizeof(float *));
|
||||
m->exp_w2[l] = calloc((size_t)c->moe_n_experts, sizeof(float *));
|
||||
if (c->moe_mask[l]) {
|
||||
m->router[l] = malloc((size_t)c->d_model * c->moe_n_experts * sizeof(float));
|
||||
} else {
|
||||
m->ffn_w1[l] = malloc((size_t)c->d_model * c->d_ff * sizeof(float));
|
||||
m->ffn_w2[l] = malloc((size_t)c->d_ff * c->d_model * sizeof(float));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int model_load(Model *m, const char *path) {
|
||||
if (!m || !path) return -1;
|
||||
memset(m, 0, sizeof(*m));
|
||||
|
||||
FILE *fh = fopen(path, "rb");
|
||||
if (!fh) { perror("open"); return -2; }
|
||||
fseek(fh, 0, SEEK_END);
|
||||
long sz = ftell(fh);
|
||||
fseek(fh, 0, SEEK_SET);
|
||||
if (sz <= 0) { fclose(fh); return -3; }
|
||||
unsigned char *buf = malloc((size_t)sz);
|
||||
if (!buf) { fclose(fh); return -4; }
|
||||
size_t nrd = fread(buf, 1, (size_t)sz, fh);
|
||||
fclose(fh);
|
||||
if (nrd != (size_t)sz) { free(buf); return -5; }
|
||||
|
||||
size_t off = 0;
|
||||
ModelConfig *c = &m->config;
|
||||
config_init(c);
|
||||
if (expect_magic(buf, &off) != 0) { free(buf); return -10; }
|
||||
uint32_t version = rd_u32(buf, &off);
|
||||
if (version != 1) { free(buf); return -11; }
|
||||
uint32_t dtype = rd_u32(buf, &off);
|
||||
if (dtype > 2) { free(buf); return -12; } /* 0=f32,1=fp16,2=q8 */
|
||||
|
||||
c->vocab_size = (int)rd_u32(buf, &off);
|
||||
c->n_layer = (int)rd_u32(buf, &off);
|
||||
c->d_model = (int)rd_u32(buf, &off);
|
||||
c->n_head = (int)rd_u32(buf, &off);
|
||||
c->d_ff = (int)rd_u32(buf, &off);
|
||||
c->moe_n_experts = (int)rd_u32(buf, &off);
|
||||
c->moe_top_k = (int)rd_u32(buf, &off);
|
||||
c->d_expert = (int)rd_u32(buf, &off);
|
||||
c->max_seq_len = (int)rd_u32(buf, &off);
|
||||
c->num_merges = (int)rd_u32(buf, &off);
|
||||
c->n_special = (int)rd_u32(buf, &off);
|
||||
c->tie_weights = buf[off]; off += 4; /* 1 byte + 3 reserve */
|
||||
c->rmsnorm_eps = rd_f32(buf, &off);
|
||||
c->head_dim = c->d_model / c->n_head;
|
||||
|
||||
/* moe_mask */
|
||||
c->moe_mask = malloc((size_t)c->n_layer);
|
||||
memcpy(c->moe_mask, buf + off, (size_t)c->n_layer);
|
||||
off += (size_t)c->n_layer;
|
||||
|
||||
/* ---- vocab blob ---- */
|
||||
uint32_t blob_len = rd_u32(buf, &off);
|
||||
size_t boff = 0;
|
||||
const unsigned char *blob = buf + off;
|
||||
off += blob_len;
|
||||
|
||||
unsigned char **vbytes = calloc((size_t)c->vocab_size, sizeof(unsigned char *));
|
||||
int *vlen = calloc((size_t)c->vocab_size, sizeof(int));
|
||||
for (int i = 0; i < c->vocab_size; i++) {
|
||||
uint32_t tl = rd_u32(blob, &boff);
|
||||
vbytes[i] = (unsigned char *)(blob + boff); /* 指向 buffer 内 */
|
||||
vlen[i] = (int)tl;
|
||||
boff += tl;
|
||||
}
|
||||
uint32_t *ma = malloc((size_t)(c->num_merges > 0 ? c->num_merges : 1) * sizeof(uint32_t));
|
||||
uint32_t *mb = malloc((size_t)(c->num_merges > 0 ? c->num_merges : 1) * sizeof(uint32_t));
|
||||
for (int i = 0; i < c->num_merges; i++) {
|
||||
ma[i] = rd_u32(blob, &boff);
|
||||
mb[i] = rd_u32(blob, &boff);
|
||||
}
|
||||
tokenizer_build(&m->tok, c->vocab_size, (const unsigned char **)vbytes, vlen,
|
||||
c->num_merges, ma, mb);
|
||||
free(vbytes);
|
||||
free(vlen);
|
||||
free(ma);
|
||||
free(mb);
|
||||
|
||||
/* ---- weights 段 ---- */
|
||||
uint32_t n_tensors = rd_u32(buf, &off);
|
||||
|
||||
if (alloc_weights(m) != 0) { free(buf); return -14; }
|
||||
|
||||
for (uint32_t t = 0; t < n_tensors; t++) {
|
||||
uint32_t nl = rd_u32(buf, &off);
|
||||
char name[128];
|
||||
if (nl >= sizeof(name)) nl = sizeof(name) - 1;
|
||||
memcpy(name, buf + off, nl);
|
||||
name[nl] = '\0';
|
||||
off += nl;
|
||||
uint32_t rows = rd_u32(buf, &off);
|
||||
uint32_t cols = rd_u32(buf, &off);
|
||||
size_t n = (size_t)rows * cols;
|
||||
float *tmp = malloc(n * sizeof(float));
|
||||
if (!tmp) { free(buf); return -13; }
|
||||
size_t used = read_weight_f32(buf + off, rows, cols, dtype, tmp);
|
||||
off += used;
|
||||
load_weight(m, name, rows, cols, tmp);
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
m->is_loaded = 1;
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* 从 GGUF 加载。要求使用我们定义的元数据键（见 platform/gguf_format.py）*/
|
||||
int model_load_gguf(Model *m, const char *path) {
|
||||
if (!m || !path) return -1;
|
||||
memset(m, 0, sizeof(*m));
|
||||
|
||||
Gguf g;
|
||||
if (gguf_open(&g, path) != 0) { fprintf(stderr, "[err] 无法解析 GGUF: %s\n", path); return -20; }
|
||||
|
||||
ModelConfig *c = &m->config;
|
||||
config_init(c);
|
||||
|
||||
uint32_t u;
|
||||
if (!gguf_meta_u32(&g, "pap.vocab_size", &u)) { gguf_close(&g); return -21; } c->vocab_size = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.n_layer", &u)) { gguf_close(&g); return -21; } c->n_layer = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.d_model", &u)) { gguf_close(&g); return -21; } c->d_model = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.n_head", &u)) { gguf_close(&g); return -21; } c->n_head = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.d_ff", &u)) { gguf_close(&g); return -21; } c->d_ff = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.moe_n_experts", &u)) { gguf_close(&g); return -21; } c->moe_n_experts = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.moe_top_k", &u)) { gguf_close(&g); return -21; } c->moe_top_k = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.d_expert", &u)) { gguf_close(&g); return -21; } c->d_expert = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.max_seq_len", &u)) { gguf_close(&g); return -21; } c->max_seq_len = (int)u;
|
||||
if (!gguf_meta_u32(&g, "pap.num_merges", &u)) { gguf_close(&g); return -21; } c->num_merges = (int)u;
|
||||
gguf_meta_f32(&g, "pap.rmsnorm_eps", &c->rmsnorm_eps);
|
||||
c->head_dim = c->d_model / c->n_head;
|
||||
|
||||
/* moe_mask */
|
||||
if (c->n_layer > 0) {
|
||||
c->moe_mask = malloc((size_t)c->n_layer);
|
||||
uint64_t got = (uint64_t)c->n_layer;
|
||||
if (!gguf_meta_u8_array(&g, "pap.moe_mask", c->moe_mask, got)) {
|
||||
/* 缺省：全部 MoE */
|
||||
for (int l = 0; l < c->n_layer; l++) c->moe_mask[l] = (c->moe_n_experts > 0) ? 1 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (alloc_weights(m) != 0) { gguf_close(&g); return -22; }
|
||||
|
||||
/* 逐张量反量化并按名装载（未知名字会因精确匹配被忽略） */
|
||||
for (uint64_t i = 0; i < g.tensor_count; i++) {
|
||||
uint64_t n = 1;
|
||||
for (uint32_t d = 0; d < g.n_dims[i]; d++) n *= g.dims[i][d];
|
||||
float *tmp = malloc((size_t)n * sizeof(float));
|
||||
if (gguf_tensor_to_f32(&g, g.tensor_name[i], tmp) > 0) {
|
||||
load_weight(m, g.tensor_name[i], 1, (uint32_t)n, tmp);
|
||||
}
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
/* 词表 + merges（从元数据还原） */
|
||||
if (c->vocab_size > 0) {
|
||||
uint32_t *vlen = calloc((size_t)c->vocab_size, sizeof(uint32_t));
|
||||
uint64_t blob_len = 0;
|
||||
if (gguf_meta_u32_array(&g, "pap.vocab_len", vlen, (uint64_t)c->vocab_size)) {
|
||||
for (int i = 0; i < c->vocab_size; i++) blob_len += vlen[i];
|
||||
uint8_t *blob = malloc((size_t)blob_len);
|
||||
if (gguf_meta_u8_array(&g, "pap.vocab_blob", blob, blob_len)) {
|
||||
unsigned char **vbytes = calloc((size_t)c->vocab_size, sizeof(unsigned char *));
|
||||
int *bytes = calloc((size_t)c->vocab_size, sizeof(int));
|
||||
size_t off = 0;
|
||||
for (int i = 0; i < c->vocab_size; i++) {
|
||||
vbytes[i] = blob + off;
|
||||
bytes[i] = (int)vlen[i];
|
||||
off += vlen[i];
|
||||
}
|
||||
uint32_t *ma = calloc((size_t)(c->num_merges > 0 ? c->num_merges : 1), sizeof(uint32_t));
|
||||
uint32_t *mb = calloc((size_t)(c->num_merges > 0 ? c->num_merges : 1), sizeof(uint32_t));
|
||||
if (c->num_merges > 0) {
|
||||
gguf_meta_u32_array(&g, "pap.merges_a", ma, (uint64_t)c->num_merges);
|
||||
gguf_meta_u32_array(&g, "pap.merges_b", mb, (uint64_t)c->num_merges);
|
||||
}
|
||||
tokenizer_build(&m->tok, c->vocab_size, (const unsigned char **)vbytes, bytes,
|
||||
c->num_merges, ma, mb);
|
||||
free(vbytes); free(bytes); free(ma); free(mb);
|
||||
}
|
||||
free(blob);
|
||||
}
|
||||
free(vlen);
|
||||
}
|
||||
|
||||
gguf_close(&g);
|
||||
m->is_loaded = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void model_free(Model *m) {
|
||||
if (!m) return;
|
||||
int L = m->config.n_layer;
|
||||
tokenizer_free(&m->tok);
|
||||
free(m->wte);
|
||||
free(m->wpe);
|
||||
for (int l = 0; l < L; l++) {
|
||||
free(m->l_norm1[l]);
|
||||
free(m->wq[l]);
|
||||
free(m->wk[l]);
|
||||
free(m->wv[l]);
|
||||
free(m->wo[l]);
|
||||
free(m->l_norm2[l]);
|
||||
free(m->router[l]);
|
||||
if (m->exp_w1[l]) {
|
||||
for (int e = 0; e < m->config.moe_n_experts; e++) {
|
||||
free(m->exp_w1[l][e]);
|
||||
free(m->exp_w2[l][e]);
|
||||
}
|
||||
}
|
||||
free(m->exp_w1[l]);
|
||||
free(m->exp_w2[l]);
|
||||
free(m->ffn_w1[l]);
|
||||
free(m->ffn_w2[l]);
|
||||
}
|
||||
free(m->l_norm1);
|
||||
free(m->wq);
|
||||
free(m->wk);
|
||||
free(m->wv);
|
||||
free(m->wo);
|
||||
free(m->l_norm2);
|
||||
free(m->router);
|
||||
free(m->exp_w1);
|
||||
free(m->exp_w2);
|
||||
free(m->ffn_w1);
|
||||
free(m->ffn_w2);
|
||||
free(m->norm_final);
|
||||
free(m->lm_head);
|
||||
config_free(&m->config);
|
||||
memset(m, 0, sizeof(*m));
|
||||
}
|
||||
|
||||
int64_t model_param_count(const Model *m) {
|
||||
const ModelConfig *c = &m->config;
|
||||
int64_t n = 0;
|
||||
n += (int64_t)c->vocab_size * c->d_model; /* wte */
|
||||
n += (int64_t)c->max_seq_len * c->d_model; /* wpe */
|
||||
for (int l = 0; l < c->n_layer; l++) {
|
||||
n += c->d_model; /* norm1 */
|
||||
n += 4LL * c->d_model * c->d_model; /* qkv+o */
|
||||
n += c->d_model; /* norm2 */
|
||||
if (c->moe_mask[l]) {
|
||||
n += (int64_t)c->d_model * c->moe_n_experts; /* router */
|
||||
n += 2LL * c->moe_n_experts * c->d_model * c->d_expert;
|
||||
} else {
|
||||
n += 2LL * c->d_model * c->d_ff;
|
||||
}
|
||||
}
|
||||
n += c->d_model; /* norm_final */
|
||||
n += (int64_t)c->vocab_size * c->d_model; /* lm_head */
|
||||
return n;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
#ifndef MO_MODEL_H
|
||||
#define MO_MODEL_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "model/config.h"
|
||||
#include "model/tokenizer.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* 已加载的模型：config + tokenizer + 全部权重（float32，行主序，列为 (in, out)）。 */
|
||||
typedef struct {
|
||||
ModelConfig config;
|
||||
Tokenizer tok;
|
||||
|
||||
/* 嵌入 */
|
||||
float *wte; /* vocab * d_model */
|
||||
float *wpe; /* max_seq * d_model */
|
||||
|
||||
/* 每层 */
|
||||
float **l_norm1; /* [layer] d_model */
|
||||
float **wq, **wk, **wv, **wo; /* [layer] d_model*d_model */
|
||||
float **l_norm2; /* [layer] d_model */
|
||||
float **router; /* [layer] d_model*n_experts；稠密层为 NULL */
|
||||
float ***exp_w1; /* [layer][expert] d_model*d_expert */
|
||||
float ***exp_w2; /* [layer][expert] d_expert*d_model */
|
||||
float **ffn_w1; /* [layer] d_model*d_ff；MoE 层为 NULL */
|
||||
float **ffn_w2; /* [layer] d_ff*d_model */
|
||||
|
||||
/* 最终 */
|
||||
float *norm_final; /* d_model */
|
||||
float *lm_head; /* vocab * d_model */
|
||||
|
||||
int is_loaded;
|
||||
} Model;
|
||||
|
||||
/* 从 .pap 文件加载模型。成功返回 0 */
|
||||
int model_load(Model *m, const char *path);
|
||||
/* 从 GGUF 文件加载模型（使用我们定义的元数据键）。成功返回 0 */
|
||||
int model_load_gguf(Model *m, const char *path);
|
||||
void model_free(Model *m);
|
||||
int64_t model_param_count(const Model *m);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_MODEL_H */
|
||||
@@ -0,0 +1,109 @@
|
||||
#include "model/tokenizer.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
int tokenizer_build(Tokenizer *t,
|
||||
int vocab_size, const unsigned char **vocab, const int *vocab_len,
|
||||
int num_merges, const uint32_t *merge_a, const uint32_t *merge_b) {
|
||||
if (!t || vocab_size <= 0) return -1;
|
||||
t->vocab_size = vocab_size;
|
||||
t->num_merges = num_merges;
|
||||
|
||||
t->vocab = calloc((size_t)vocab_size, sizeof(unsigned char *));
|
||||
t->vocab_len = calloc((size_t)vocab_size, sizeof(int));
|
||||
if (!t->vocab || !t->vocab_len) return -1;
|
||||
for (int i = 0; i < vocab_size; i++) {
|
||||
t->vocab[i] = malloc((size_t)(vocab_len[i] ? vocab_len[i] : 1));
|
||||
if (!t->vocab[i]) return -1;
|
||||
memcpy(t->vocab[i], vocab[i], (size_t)vocab_len[i]);
|
||||
t->vocab_len[i] = vocab_len[i];
|
||||
}
|
||||
|
||||
if (num_merges > 0) {
|
||||
t->merge_a = malloc((size_t)num_merges * sizeof(uint32_t));
|
||||
t->merge_b = malloc((size_t)num_merges * sizeof(uint32_t));
|
||||
if (!t->merge_a || !t->merge_b) return -1;
|
||||
memcpy(t->merge_a, merge_a, (size_t)num_merges * sizeof(uint32_t));
|
||||
memcpy(t->merge_b, merge_b, (size_t)num_merges * sizeof(uint32_t));
|
||||
} else {
|
||||
t->merge_a = NULL;
|
||||
t->merge_b = NULL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tokenizer_free(Tokenizer *t) {
|
||||
if (!t) return;
|
||||
if (t->vocab) {
|
||||
for (int i = 0; i < t->vocab_size; i++) free(t->vocab[i]);
|
||||
free(t->vocab);
|
||||
}
|
||||
free(t->vocab_len);
|
||||
free(t->merge_a);
|
||||
free(t->merge_b);
|
||||
memset(t, 0, sizeof(*t));
|
||||
}
|
||||
|
||||
/* 找到 id 下标 [st, ed] 的字节拼接长度，用于 rank 遍历时比对。这里直接比较 merge 的 (a,b) 与当前 token 的 (id[i],id[i+1])。 */
|
||||
static int merge_rank(const Tokenizer *t, uint32_t a, uint32_t b) {
|
||||
for (int i = 0; i < t->num_merges; i++) {
|
||||
if (t->merge_a[i] == a && t->merge_b[i] == b) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
int tokenizer_encode(const Tokenizer *t, const char *text, int32_t *out_ids, int max_out) {
|
||||
if (!t || !text || !out_ids) return -1;
|
||||
size_t n = strlen(text);
|
||||
if (n == 0) return 0;
|
||||
|
||||
/* 每个字节一个初始 token，id=字节值(0..255) */
|
||||
int *tok = malloc((size_t)(n + 1) * sizeof(int));
|
||||
if (!tok) return -1;
|
||||
int len = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
tok[len++] = (unsigned char)text[i];
|
||||
}
|
||||
|
||||
const int base = 256;
|
||||
while (1) {
|
||||
int best = -1, best_rank = 0x7fffffff;
|
||||
for (int i = 0; i + 1 < len; i++) {
|
||||
int r = merge_rank(t, (uint32_t)tok[i], (uint32_t)tok[i + 1]);
|
||||
if (r >= 0 && r < best_rank) {
|
||||
best_rank = r;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
if (best < 0) break;
|
||||
tok[best] = base + best_rank;
|
||||
memmove(&tok[best + 1], &tok[best + 2], (size_t)(len - best - 2) * sizeof(int));
|
||||
len--;
|
||||
}
|
||||
|
||||
if (len > max_out) {
|
||||
free(tok);
|
||||
return -2;
|
||||
}
|
||||
for (int i = 0; i < len; i++) out_ids[i] = tok[i];
|
||||
free(tok);
|
||||
return len;
|
||||
}
|
||||
|
||||
int tokenizer_decode(const Tokenizer *t, const int32_t *ids, int n, char *out, int max_out) {
|
||||
if (!t || !ids || !out) return -1;
|
||||
int w = 0;
|
||||
for (int i = 0; i < n && w < max_out; i++) {
|
||||
int id = ids[i];
|
||||
if (id < 0 || id >= t->vocab_size || !t->vocab[id]) continue;
|
||||
int L = t->vocab_len[id];
|
||||
for (int j = 0; j < L && w < max_out; j++) {
|
||||
/* 简单替换非法 UTF-8 连续字节为 '?' 的占位处理：原样拷贝，由读取端决定 */
|
||||
out[w++] = (char)t->vocab[id][j];
|
||||
}
|
||||
}
|
||||
if (w < max_out) out[w] = '\0';
|
||||
else out[max_out - 1] = '\0';
|
||||
return w;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
#ifndef MO_TOKENIZER_H
|
||||
#define MO_TOKENIZER_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* byte-level BPE tokenizer（与 platform/tokenizer.py 完全一致）：
|
||||
* id 0..255 = 单个字节 bytes([b])
|
||||
* id 256+i = 第 i 条 merge 合并出的 token
|
||||
* merges[i] = (a,b) -> 新 id = 256 + i
|
||||
* 无特殊 token（n_special=0），直接对字节做 BPE。
|
||||
*/
|
||||
typedef struct {
|
||||
int vocab_size;
|
||||
int num_merges;
|
||||
unsigned char **vocab; /* vocab_size 个字符串 */
|
||||
int *vocab_len;
|
||||
uint32_t *merge_a; /* num_merges */
|
||||
uint32_t *merge_b;
|
||||
} Tokenizer;
|
||||
|
||||
/* 用外部数据深拷贝构建（vocab/vocab_len/merge_a/merge_b 会被复制走） */
|
||||
int tokenizer_build(Tokenizer *t,
|
||||
int vocab_size, const unsigned char **vocab, const int *vocab_len,
|
||||
int num_merges, const uint32_t *merge_a, const uint32_t *merge_b);
|
||||
|
||||
void tokenizer_free(Tokenizer *t);
|
||||
|
||||
/* 编码 text -> ids（最多 max_out 个）。返回 token 数，负数为错 */
|
||||
int tokenizer_encode(const Tokenizer *t, const char *text, int32_t *out_ids, int max_out);
|
||||
|
||||
/* 解码 ids -> utf-8 文本（最多 max_out 字节）。返回写入字节数 */
|
||||
int tokenizer_decode(const Tokenizer *t, const int32_t *ids, int n, char *out, int max_out);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_TOKENIZER_H */
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
#include "server/api.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <jansson.h>
|
||||
#include <microhttpd.h>
|
||||
|
||||
#include "infer/generate.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
/* 简单监控计数（单线程 demo 用） */
|
||||
static unsigned long g_requests = 0;
|
||||
static unsigned long g_tokens = 0;
|
||||
|
||||
/* ---- 请求上下文 ---- */
|
||||
typedef struct {
|
||||
char *body;
|
||||
size_t len;
|
||||
} ConnInfo;
|
||||
|
||||
/* ---- JSON 字符串转义（写入 dst） ---- */
|
||||
static size_t json_escape(const char *src, char *dst, size_t cap) {
|
||||
size_t w = 0;
|
||||
for (const unsigned char *p = (const unsigned char *)src; *p && w + 8 < cap; p++) {
|
||||
unsigned char c = *p;
|
||||
if (c == '"') { dst[w++] = '\\'; dst[w++] = '"'; }
|
||||
else if (c == '\\') { dst[w++] = '\\'; dst[w++] = '\\'; }
|
||||
else if (c == '\n') { dst[w++] = '\\'; dst[w++] = 'n'; }
|
||||
else if (c == '\r') { dst[w++] = '\\'; dst[w++] = 'r'; }
|
||||
else if (c == '\t') { dst[w++] = '\\'; dst[w++] = 't'; }
|
||||
else if (c < 0x20) { dst[w++] = ' '; }
|
||||
else { dst[w++] = (char)c; }
|
||||
}
|
||||
dst[w] = '\0';
|
||||
return w;
|
||||
}
|
||||
|
||||
int moe_parse_generate_request(const char *json, char *prompt, size_t prompt_cap,
|
||||
int *max_new, float *temperature, int *top_k) {
|
||||
json_error_t err;
|
||||
json_t *root = json_loads(json, 0, &err);
|
||||
if (!root || !json_is_object(root)) {
|
||||
if (root) json_decref(root);
|
||||
return -1;
|
||||
}
|
||||
int rc = 0;
|
||||
json_t *p = json_object_get(root, "prompt");
|
||||
json_t *opts = json_object_get(root, "options");
|
||||
if (!p || !json_is_string(p)) { rc = -2; goto done; }
|
||||
const char *s = json_string_value(p);
|
||||
size_t n = strlen(s);
|
||||
if (n >= prompt_cap) n = prompt_cap - 1;
|
||||
memcpy(prompt, s, n);
|
||||
prompt[n] = '\0';
|
||||
|
||||
if (opts && json_is_object(opts)) {
|
||||
json_t *mt = json_object_get(opts, "max_tokens");
|
||||
if (mt && json_is_integer(mt)) *max_new = (int)json_integer_value(mt);
|
||||
json_t *tmp = json_object_get(opts, "temperature");
|
||||
if (tmp && json_is_real(tmp)) *temperature = (float)json_real_value(tmp);
|
||||
json_t *tk = json_object_get(opts, "top_k");
|
||||
if (tk && json_is_integer(tk)) *top_k = (int)json_integer_value(tk);
|
||||
}
|
||||
done:
|
||||
json_decref(root);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- SSE 回调：逐个 token 拉取并写入 SSE 行（返回写入字节数 / 结束标记） ---- */
|
||||
static ssize_t sse_cb(void *cls, uint64_t pos, char *buf, size_t max) {
|
||||
(void)pos;
|
||||
GenStream *gs = (GenStream *)cls;
|
||||
size_t w = 0;
|
||||
char tok[64];
|
||||
char esc[256];
|
||||
|
||||
while (w + 96 < max) {
|
||||
int r = gen_stream_next(gs, tok, sizeof(tok));
|
||||
if (r <= 0) {
|
||||
/* 先冲掉已缓冲数据；否则结束流 */
|
||||
return (ssize_t)(w > 0 ? w : MHD_CONTENT_READER_END_OF_STREAM);
|
||||
}
|
||||
g_tokens++; /* 监控：累计生成 token */
|
||||
json_escape(tok, esc, sizeof(esc));
|
||||
int n = snprintf(buf + w, max - w, "data: {\"response\":\"%s\"}\n\n", esc);
|
||||
w += (size_t)n;
|
||||
}
|
||||
return (ssize_t)w;
|
||||
}
|
||||
|
||||
static void sse_free(void *cls) {
|
||||
GenStream *gs = (GenStream *)cls;
|
||||
if (gs) {
|
||||
gen_stream_end(gs);
|
||||
free(gs);
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- 连接完成时释放 ConnInfo ---- */
|
||||
static void completion_cb(void *cls, struct MHD_Connection *conn, void **con_cls,
|
||||
enum MHD_RequestTerminationCode code) {
|
||||
(void)cls; (void)conn; (void)code;
|
||||
ConnInfo *ci = *con_cls;
|
||||
if (ci) {
|
||||
free(ci->body);
|
||||
free(ci);
|
||||
*con_cls = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static char *read_body(ConnInfo *ci, const char *upload, size_t *upload_size) {
|
||||
if (*upload_size == 0) return NULL;
|
||||
ci->body = realloc(ci->body, ci->len + *upload_size + 1);
|
||||
memcpy(ci->body + ci->len, upload, *upload_size);
|
||||
ci->len += *upload_size;
|
||||
ci->body[ci->len] = '\0';
|
||||
size_t n = *upload_size;
|
||||
*upload_size = 0;
|
||||
(void)n;
|
||||
return ci->body;
|
||||
}
|
||||
|
||||
static enum MHD_Result send_text(struct MHD_Connection *conn, unsigned status, const char *ct, const char *body) {
|
||||
struct MHD_Response *resp = MHD_create_response_from_buffer(strlen(body), (void *)body, MHD_RESPMEM_MUST_COPY);
|
||||
if (!resp) return MHD_NO;
|
||||
MHD_add_response_header(resp, "Content-Type", ct);
|
||||
enum MHD_Result ret = MHD_queue_response(conn, status, resp);
|
||||
MHD_destroy_response(resp);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static enum MHD_Result access_handler(void *cls, struct MHD_Connection *conn,
|
||||
const char *url, const char *method,
|
||||
const char *version, const char *upload_data,
|
||||
size_t *upload_data_size, void **con_cls) {
|
||||
(void)version;
|
||||
const Model *m = (const Model *)cls;
|
||||
ConnInfo *ci = (ConnInfo *)*con_cls;
|
||||
if (!ci) {
|
||||
ci = calloc(1, sizeof(ConnInfo));
|
||||
*con_cls = ci;
|
||||
return MHD_YES;
|
||||
}
|
||||
|
||||
/* 读取 POST 请求体 */
|
||||
if (strcmp(method, MHD_HTTP_METHOD_POST) == 0) {
|
||||
if (*upload_data_size) {
|
||||
read_body(ci, upload_data, upload_data_size);
|
||||
return MHD_YES;
|
||||
}
|
||||
}
|
||||
|
||||
/* GET /health */
|
||||
if (strcmp(method, MHD_HTTP_METHOD_GET) == 0 && strcmp(url, "/health") == 0)
|
||||
return send_text(conn, MHD_HTTP_OK, "text/plain", "ok");
|
||||
|
||||
/* GET /api/models */
|
||||
if (strcmp(method, MHD_HTTP_METHOD_GET) == 0 && strcmp(url, "/api/models") == 0) {
|
||||
char buf[256];
|
||||
snprintf(buf, sizeof(buf),
|
||||
"{\"model\":\"moe-0.1b\",\"params\":%lld,\"arch\":\"moe-pap\",\"n_experts\":%d}",
|
||||
(long long)model_param_count(m), m->config.moe_n_experts);
|
||||
return send_text(conn, MHD_HTTP_OK, "application/json", buf);
|
||||
}
|
||||
|
||||
/* GET /api/metrics */
|
||||
if (strcmp(method, MHD_HTTP_METHOD_GET) == 0 && strcmp(url, "/api/metrics") == 0) {
|
||||
char buf[128];
|
||||
snprintf(buf, sizeof(buf), "{\"requests\":%lu,\"tokens\":%lu}", g_requests, g_tokens);
|
||||
return send_text(conn, MHD_HTTP_OK, "application/json", buf);
|
||||
}
|
||||
|
||||
/* POST /api/generate (SSE) */
|
||||
if (strcmp(method, MHD_HTTP_METHOD_POST) == 0 && strcmp(url, "/api/generate") == 0) {
|
||||
g_requests++;
|
||||
if (!ci->body) return send_text(conn, MHD_HTTP_BAD_REQUEST, "text/plain", "empty body");
|
||||
char prompt[2048];
|
||||
int max_new = 100, top_k = 40;
|
||||
float temperature = 0.8f;
|
||||
int rc = moe_parse_generate_request(ci->body, prompt, sizeof(prompt),
|
||||
&max_new, &temperature, &top_k);
|
||||
if (rc != 0) return send_text(conn, MHD_HTTP_BAD_REQUEST, "text/plain", "bad json");
|
||||
|
||||
GenStream *gs = calloc(1, sizeof(GenStream));
|
||||
if (gen_stream_begin(gs, m, prompt, max_new, temperature, top_k) < 0) {
|
||||
free(gs);
|
||||
return send_text(conn, MHD_HTTP_INTERNAL_SERVER_ERROR, "text/plain", "gen init fail");
|
||||
}
|
||||
struct MHD_Response *resp = MHD_create_response_from_callback(MHD_SIZE_UNKNOWN, 4096, &sse_cb, gs, &sse_free);
|
||||
if (!resp) { gen_stream_end(gs); free(gs); return MHD_NO; }
|
||||
MHD_add_response_header(resp, "Content-Type", "text/event-stream");
|
||||
MHD_add_response_header(resp, "Cache-Control", "no-cache");
|
||||
enum MHD_Result ret = MHD_queue_response(conn, MHD_HTTP_OK, resp);
|
||||
MHD_destroy_response(resp);
|
||||
return ret;
|
||||
}
|
||||
|
||||
return send_text(conn, MHD_HTTP_NOT_FOUND, "text/plain", "not found");
|
||||
}
|
||||
|
||||
int moe_server_run(const Model *m, const char *model_name, unsigned port) {
|
||||
(void)model_name;
|
||||
struct MHD_Daemon *daemon = MHD_start_daemon(
|
||||
MHD_USE_THREAD_PER_CONNECTION,
|
||||
(unsigned short)port, NULL, NULL, &access_handler, (void *)m,
|
||||
MHD_OPTION_NOTIFY_COMPLETED, &completion_cb, NULL,
|
||||
MHD_OPTION_END);
|
||||
if (!daemon) {
|
||||
MO_LOGE("HTTP 服务启动失败: %u", port);
|
||||
return 1;
|
||||
}
|
||||
MO_LOGI("pmai HTTP 已启动: http://127.0.0.1:%u 模型=%s", port, model_name);
|
||||
MO_LOGI(" GET /health");
|
||||
MO_LOGI(" GET /api/models");
|
||||
MO_LOGI(" GET /api/metrics");
|
||||
MO_LOGI(" POST /api/generate (SSE) Ctrl-C 退出");
|
||||
for (;;) sleep(3600); /* 阻塞直到 Ctrl-C 终止进程 */
|
||||
return 0;
|
||||
}
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
#ifndef MO_API_H
|
||||
#define MO_API_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#include "model/model.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* 启动 HTTP 服务并阻塞。返回 0 正常退出，非0为失败。 */
|
||||
int moe_server_run(const Model *m, const char *model_name, unsigned port);
|
||||
|
||||
/* 供测试：解析 JSON 请求体为 (prompt, max_new, temperature, top_k)。 */
|
||||
/* 返回 0 成功，非0 失败。 */
|
||||
int moe_parse_generate_request(const char *json, char *prompt, size_t prompt_cap,
|
||||
int *max_new, float *temperature, int *top_k);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_API_H */
|
||||
@@ -0,0 +1,46 @@
|
||||
#include "utils/logger.h"
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <time.h>
|
||||
|
||||
static MoLogLevel g_level = MO_LOG_INFO;
|
||||
static int g_timestamps = 1;
|
||||
|
||||
static const char *level_name(MoLogLevel l) {
|
||||
switch (l) {
|
||||
case MO_LOG_DEBUG: return "DEBUG";
|
||||
case MO_LOG_INFO: return "INFO";
|
||||
case MO_LOG_WARN: return "WARN";
|
||||
case MO_LOG_ERROR: return "ERROR";
|
||||
default: return "????";
|
||||
}
|
||||
}
|
||||
|
||||
void mo_log_set_level(MoLogLevel level) { g_level = level; }
|
||||
MoLogLevel mo_log_get_level(void) { return g_level; }
|
||||
void mo_log_set_timestamps(int enabled) { g_timestamps = enabled; }
|
||||
|
||||
void mo_log_msg(MoLogLevel level, const char *fmt, ...) {
|
||||
if (level < g_level) return;
|
||||
|
||||
FILE *out = (level >= MO_LOG_WARN) ? stderr : stdout;
|
||||
|
||||
if (g_timestamps) {
|
||||
time_t t = time(NULL);
|
||||
struct tm tm_buf;
|
||||
char ts[32];
|
||||
localtime_r(&t, &tm_buf); /* POSIX；Linux/glibc 下可用 */
|
||||
strftime(ts, sizeof(ts), "%Y-%m-%d %H:%M:%S", &tm_buf);
|
||||
fprintf(out, "%s [%5s] ", ts, level_name(level));
|
||||
} else {
|
||||
fprintf(out, "[%5s] ", level_name(level));
|
||||
}
|
||||
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vfprintf(out, fmt, args);
|
||||
va_end(args);
|
||||
fputc('\n', out);
|
||||
fflush(out);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#ifndef MO_LOGGER_H
|
||||
#define MO_LOGGER_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* 日志级别 */
|
||||
typedef enum {
|
||||
MO_LOG_DEBUG = 0,
|
||||
MO_LOG_INFO,
|
||||
MO_LOG_WARN,
|
||||
MO_LOG_ERROR,
|
||||
MO_LOG_NONE
|
||||
} MoLogLevel;
|
||||
|
||||
/* 控制全局日志级别 */
|
||||
void mo_log_set_level(MoLogLevel level);
|
||||
MoLogLevel mo_log_get_level(void);
|
||||
|
||||
/* 是否打印时间戳（默认开） */
|
||||
void mo_log_set_timestamps(int enabled);
|
||||
|
||||
/* 一条日志；内部追加换行。level 低于全局级别时忽略 */
|
||||
void mo_log_msg(MoLogLevel level, const char *fmt, ...)
|
||||
#if defined(__GNUC__)
|
||||
__attribute__((format(printf, 2, 3)))
|
||||
#endif
|
||||
;
|
||||
|
||||
#define MO_LOGD(...) mo_log_msg(MO_LOG_DEBUG, __VA_ARGS__)
|
||||
#define MO_LOGI(...) mo_log_msg(MO_LOG_INFO, __VA_ARGS__)
|
||||
#define MO_LOGW(...) mo_log_msg(MO_LOG_WARN, __VA_ARGS__)
|
||||
#define MO_LOGE(...) mo_log_msg(MO_LOG_ERROR, __VA_ARGS__)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_LOGGER_H */
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#include "utils/math.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
float mo_tanhf(float x) { return tanhf(x); }
|
||||
|
||||
float mo_sigmoidf(float x) {
|
||||
/* 数值稳定：避免 expf 溢出 */
|
||||
if (x >= 0.0f) {
|
||||
float e = expf(-x);
|
||||
return 1.0f / (1.0f + e);
|
||||
} else {
|
||||
float e = expf(x);
|
||||
return e / (1.0f + e);
|
||||
}
|
||||
}
|
||||
|
||||
float mo_gelu(float x) {
|
||||
/* GELU 的 tanh 近似：0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 x^3))) */
|
||||
const float c = 0.7978845608028654f; /* sqrt(2/pi) */
|
||||
const float a = 0.044715f;
|
||||
return 0.5f * x * (1.0f + tanhf(c * (x + a * x * x * x)));
|
||||
}
|
||||
|
||||
float mo_fast_exp(float x) {
|
||||
/* Schraudolph 近似；Range 约为 (-88, 88)，略偏大，适合做 softmax 相对比较 */
|
||||
union { float f; int32_t i; } v;
|
||||
v.i = (int32_t)(12102203.0f * x + 1065353216.0f);
|
||||
return v.f;
|
||||
}
|
||||
|
||||
float mo_fast_sigmoid(float x) {
|
||||
float e = mo_fast_exp(-x);
|
||||
return 1.0f / (1.0f + e);
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
#ifndef MO_MATH_H
|
||||
#define MO_MATH_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ---- 神经网络常用激活函数（标准库实现，精度优先） ---- */
|
||||
|
||||
/* tanh */
|
||||
float mo_tanhf(float x);
|
||||
|
||||
/* sigmoid = 1 / (1 + exp(-x)) */
|
||||
float mo_sigmoidf(float x);
|
||||
|
||||
/* GELU（GPT-2 的 tanh 近似） */
|
||||
float mo_gelu(float x);
|
||||
|
||||
/* ---- 快速近似（速度优先，精度有损） ---- */
|
||||
|
||||
/* 快速指数近似（Schraudolph 位技巧），用于需要高吞吐的路径 */
|
||||
float mo_fast_exp(float x);
|
||||
|
||||
/* 快速 sigmoid，基于 mo_fast_exp */
|
||||
float mo_fast_sigmoid(float x);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_MATH_H */
|
||||
@@ -0,0 +1,84 @@
|
||||
#include "utils/mempool.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
|
||||
#define MO_DEFAULT_BLOCK_SIZE (64u * 1024u)
|
||||
#define MO_ALIGN 64u
|
||||
|
||||
static size_t align_up(size_t n, size_t a) {
|
||||
return (n + a - 1u) & ~(a - 1u);
|
||||
}
|
||||
|
||||
MoMemPool *mempool_create(size_t block_size) {
|
||||
MoMemPool *p = calloc(1, sizeof(MoMemPool));
|
||||
if (!p) return NULL;
|
||||
p->block_size = block_size ? block_size : MO_DEFAULT_BLOCK_SIZE;
|
||||
return p;
|
||||
}
|
||||
|
||||
static MoMemPoolBlock *new_block(size_t min_cap, size_t block_size) {
|
||||
/* 块容量向上取整到对齐边界；基址用 posix_memalign 保证强对齐，
|
||||
这样返回给调用方的指针也能保持 MO_ALIGN 对齐（为 SIMD/AVX 准备）。 */
|
||||
size_t cap = align_up(min_cap > block_size ? min_cap : block_size, MO_ALIGN);
|
||||
MoMemPoolBlock *b = calloc(1, sizeof(MoMemPoolBlock));
|
||||
if (!b) return NULL;
|
||||
int rc = posix_memalign((void **)&b->base, MO_ALIGN, cap);
|
||||
if (rc != 0) {
|
||||
free(b);
|
||||
return NULL;
|
||||
}
|
||||
b->capacity = cap;
|
||||
b->used = 0;
|
||||
return b;
|
||||
}
|
||||
|
||||
void *mempool_alloc(MoMemPool *pool, size_t size) {
|
||||
if (!pool || size == 0) return NULL;
|
||||
size = align_up(size, MO_ALIGN); /* 至少 64 对齐 */
|
||||
|
||||
/* 优先复用已有块 */
|
||||
for (MoMemPoolBlock *b = pool->blocks; b; b = b->next) {
|
||||
if (b->capacity - b->used >= size) {
|
||||
void *p = b->base + b->used;
|
||||
b->used += size;
|
||||
return p;
|
||||
}
|
||||
}
|
||||
|
||||
/* 没有合适块，新建一个并头插 */
|
||||
MoMemPoolBlock *b = new_block(size, pool->block_size);
|
||||
if (!b) return NULL;
|
||||
b->next = pool->blocks;
|
||||
pool->blocks = b;
|
||||
|
||||
void *p = b->base + b->used;
|
||||
b->used += size;
|
||||
return p;
|
||||
}
|
||||
|
||||
void mempool_clear(MoMemPool *pool) {
|
||||
if (!pool) return;
|
||||
for (MoMemPoolBlock *b = pool->blocks; b; b = b->next) {
|
||||
b->used = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void mempool_destroy(MoMemPool *pool) {
|
||||
if (!pool) return;
|
||||
MoMemPoolBlock *b = pool->blocks;
|
||||
while (b) {
|
||||
MoMemPoolBlock *next = b->next;
|
||||
free(b->base);
|
||||
free(b);
|
||||
b = next;
|
||||
}
|
||||
free(pool);
|
||||
}
|
||||
|
||||
size_t mempool_used_bytes(const MoMemPool *pool) {
|
||||
size_t total = 0;
|
||||
for (const MoMemPoolBlock *b = pool->blocks; b; b = b->next) {
|
||||
total += b->used;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
#ifndef MO_MEMPOOL_H
|
||||
#define MO_MEMPOOL_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* 内存池（Arena）。
|
||||
*
|
||||
* 目的：把大量短生命周期的小分配聚合成少数大块分配，
|
||||
* 减少 malloc/free 调用次数与内存碎片化。一次性全部释放（mempool_clear/destroy）。
|
||||
*
|
||||
* 注意：不是通用 malloc 替代品 —— 不支持单个对象 free，仅供一次性、同生命周期数据使用。
|
||||
*/
|
||||
|
||||
typedef struct MoMemPoolBlock {
|
||||
uint8_t *base; /* 块起始 */
|
||||
size_t capacity; /* 块容量（字节） */
|
||||
size_t used; /* 已用（字节，对齐后） */
|
||||
struct MoMemPoolBlock *next;
|
||||
} MoMemPoolBlock;
|
||||
|
||||
typedef struct {
|
||||
size_t block_size; /* 每个新块的默认容量 */
|
||||
MoMemPoolBlock *blocks;
|
||||
} MoMemPool;
|
||||
|
||||
/* 创建内存池；block_size=0 时用默认 64KB。返回池指针，失败返回 NULL */
|
||||
MoMemPool *mempool_create(size_t block_size);
|
||||
|
||||
/* 分配 size 字节（至少 64 字节对齐）。失败返回 NULL */
|
||||
void *mempool_alloc(MoMemPool *pool, size_t size);
|
||||
|
||||
/* 清空所有已用空间（复用底层块，不释放） */
|
||||
void mempool_clear(MoMemPool *pool);
|
||||
|
||||
/* 释放整个池及其所有块 */
|
||||
void mempool_destroy(MoMemPool *pool);
|
||||
|
||||
/* 当前已用字节数 */
|
||||
size_t mempool_used_bytes(const MoMemPool *pool);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* MO_MEMPOOL_H */
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
#include "core/matrix.h"
|
||||
#include "core/tensor.h"
|
||||
#include "utils/math.h"
|
||||
#include "utils/mempool.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
static int g_pass = 0;
|
||||
static int g_fail = 0;
|
||||
|
||||
#define CHECK(cond, msg) do { \
|
||||
if (cond) { g_pass++; printf(" ok: %s\n", msg); } \
|
||||
else { g_fail++; printf(" FAIL: %s\n", msg); } \
|
||||
} while (0)
|
||||
|
||||
#define CHECK_NEAR(got, want, eps, msg) do { \
|
||||
float _g = (float)(got), _w = (float)(want); \
|
||||
if (fabsf(_g - _w) <= (eps)) { g_pass++; printf(" ok: %s (%.6f)\n", msg, _g); } \
|
||||
else { g_fail++; printf(" FAIL: %s got %.6f want %.6f\n", msg, _g, _w); } \
|
||||
} while (0)
|
||||
|
||||
static void test_tensor(void) {
|
||||
printf("[tensor]\n");
|
||||
int64_t shape[2] = {3, 4};
|
||||
MoTensor *t = tensor_create(2, shape);
|
||||
CHECK(t != NULL, "create 3x4");
|
||||
CHECK(tensor_numel(t) == 12, "numel==12");
|
||||
CHECK(t->strides[0] == 4 && t->strides[1] == 1, "row-major strides");
|
||||
|
||||
tensor_fill(t, 5.0f);
|
||||
CHECK(t->data[0] == 5.0f && t->data[11] == 5.0f, "filled all");
|
||||
|
||||
int64_t idx[2] = {2, 3};
|
||||
CHECK(tensor_offset(t, idx) == 11, "offset (2,3)==11");
|
||||
|
||||
int64_t flat[1] = {12};
|
||||
MoTensor *r = tensor_reshape(t, 1, flat);
|
||||
CHECK(r != NULL && r->ndim == 1, "reshape to 1D ok");
|
||||
int64_t bad[1] = {13};
|
||||
CHECK(tensor_reshape(t, 1, bad) == NULL, "reshape wrong size -> NULL");
|
||||
|
||||
MoTensor *clone = tensor_clone(t);
|
||||
CHECK(clone != NULL && clone->data[5] == 5.0f, "clone preserves data");
|
||||
tensor_free(clone);
|
||||
tensor_free(t);
|
||||
}
|
||||
|
||||
static void test_matmul(void) {
|
||||
printf("[matmul]\n");
|
||||
MoMat *a = mat_alloc(2, 3);
|
||||
MoMat *b = mat_alloc(3, 2);
|
||||
MoMat *c = mat_alloc(2, 2);
|
||||
float ad[6] = {1, 2, 3, 4, 5, 6};
|
||||
float bd[6] = {7, 8, 9, 10, 11, 12};
|
||||
memcpy(a->data, ad, sizeof(ad));
|
||||
memcpy(b->data, bd, sizeof(bd));
|
||||
|
||||
int rc = mat_mul(a, b, c);
|
||||
CHECK(rc == 0, "mul success");
|
||||
CHECK_NEAR(c->data[0], 58.0f, 1e-5f, "c[0][0]==58");
|
||||
CHECK_NEAR(c->data[1], 64.0f, 1e-5f, "c[0][1]==64");
|
||||
CHECK_NEAR(c->data[c->stride + 0], 139.0f, 1e-5f, "c[1][0]==139");
|
||||
CHECK_NEAR(c->data[c->stride + 1], 154.0f, 1e-5f, "c[1][1]==154");
|
||||
|
||||
MoMat *w = mat_alloc(2, 2); /* wrong N */
|
||||
CHECK(mat_mul(a, w, w) != 0, "mismatched dims -> error");
|
||||
mat_free(w);
|
||||
mat_free(a);
|
||||
mat_free(b);
|
||||
mat_free(c);
|
||||
}
|
||||
|
||||
static void test_softmax(void) {
|
||||
printf("[softmax]\n");
|
||||
MoMat *m = mat_alloc(1, 5);
|
||||
float d[5] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f};
|
||||
memcpy(m->data, d, sizeof(d));
|
||||
mat_softmax_rows(m);
|
||||
float sum = 0.0f;
|
||||
for (int j = 0; j < 5; j++) sum += m->data[j];
|
||||
CHECK_NEAR(sum, 1.0f, 1e-5f, "row sums to 1");
|
||||
CHECK(m->data[4] > m->data[0], "monotonic increasing");
|
||||
mat_free(m);
|
||||
}
|
||||
|
||||
static void test_math(void) {
|
||||
printf("[math]\n");
|
||||
CHECK_NEAR(mo_sigmoidf(0.0f), 0.5f, 1e-4f, "sigmoid(0)==0.5");
|
||||
CHECK_NEAR(mo_tanhf(0.0f), 0.0f, 1e-5f, "tanh(0)==0");
|
||||
CHECK(mo_gelu(0.0f) == 0.0f, "gelu(0)==0");
|
||||
CHECK_NEAR(mo_gelu(1.0f), 0.841192f, 1e-3f, "gelu(1) approx");
|
||||
CHECK(mo_fast_exp(1.0f) > 2.0f && mo_fast_exp(1.0f) < 3.0f, "fast_exp(1) in (2,3)");
|
||||
}
|
||||
|
||||
static void test_mempool(void) {
|
||||
printf("[mempool]\n");
|
||||
MoMemPool *pool = mempool_create(0);
|
||||
CHECK(pool != NULL, "create");
|
||||
|
||||
void *p1 = mempool_alloc(pool, 100);
|
||||
void *p2 = mempool_alloc(pool, 5000);
|
||||
CHECK(p1 != NULL && p2 != NULL, "two allocs");
|
||||
CHECK(((uintptr_t)p1 & 63u) == 0, "64-byte aligned");
|
||||
CHECK(p2 > p1, "distinct allocation");
|
||||
|
||||
/* 数据可写读回 */
|
||||
((uint8_t *)p1)[0] = 0xAB;
|
||||
((uint8_t *)p1)[99] = 0xCD;
|
||||
CHECK(((uint8_t *)p1)[0] == 0xAB && ((uint8_t *)p1)[99] == 0xCD, "read/write intact");
|
||||
|
||||
mempool_clear(pool);
|
||||
void *p3 = mempool_alloc(pool, 100);
|
||||
CHECK(p3 != NULL, "alloc after clear");
|
||||
|
||||
mempool_destroy(pool);
|
||||
CHECK(1, "destroy");
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
printf("=== moe-serve core tests ===\n");
|
||||
test_tensor();
|
||||
test_matmul();
|
||||
test_softmax();
|
||||
test_math();
|
||||
test_mempool();
|
||||
printf("=== result: %d passed, %d failed ===\n", g_pass, g_fail);
|
||||
return g_fail == 0 ? 0 : 1;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "model/gguf.h"
|
||||
#include "model/llama_tokenizer.h"
|
||||
|
||||
/* 独立验证 C 端 GPT-2 byte-level BPE 分词器（不依赖完整模型加载/架构）。 */
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) { fprintf(stderr, "用法: test_llmtok <model.gguf> [text...]\n"); return 1; }
|
||||
/* 从第 2 个参数起拼接成测试文本（若无则用默认）。 */
|
||||
char text[4096] = "Hello, world! Today is a good day.";
|
||||
if (argc > 2) { text[0] = 0; for (int i = 2; i < argc; i++) { strcat(text, argv[i]); if (i + 1 < argc) strcat(text, " "); } }
|
||||
|
||||
Gguf g;
|
||||
if (gguf_open(&g, argv[1]) != 0) { fprintf(stderr, "gguf open fail\n"); return 2; }
|
||||
char **toks = NULL; int ntok = 0;
|
||||
if (!gguf_meta_string_array(&g, "tokenizer.ggml.tokens", &toks, &ntok)) { fprintf(stderr, "no tokens meta\n"); return 3; }
|
||||
char **mrg = NULL; int nmrg = 0;
|
||||
gguf_meta_string_array(&g, "tokenizer.ggml.merges", &mrg, &nmrg);
|
||||
fprintf(stderr, "[info] vocab=%d merges=%d\n", ntok, nmrg);
|
||||
|
||||
LlamaTokenizer t;
|
||||
if (llmtok_init(&t, toks, ntok, mrg, nmrg) != 0) { fprintf(stderr, "tok init fail\n"); return 4; }
|
||||
gguf_free_string_array(toks, ntok);
|
||||
gguf_free_string_array(mrg, nmrg);
|
||||
|
||||
int ids[512];
|
||||
int n = llmtok_encode(&t, text, ids, 512);
|
||||
printf("text: %s\n", text);
|
||||
printf("ids: ");
|
||||
for (int i = 0; i < n; i++) printf("%d ", ids[i]);
|
||||
printf("\n");
|
||||
|
||||
char buf[4096];
|
||||
int b = llmtok_decode(&t, ids, n, buf, sizeof(buf));
|
||||
printf("round-trip: %.*s\n", b < 0 ? 0 : b, buf);
|
||||
|
||||
llmtok_free(&t);
|
||||
gguf_close(&g);
|
||||
return 0;
|
||||
}
|
||||
在新工单中引用
屏蔽一个用户