27 lines
858 B
Rust
27 lines
858 B
Rust
mod tokenizers;
|
|
|
|
use std::path::PathBuf;
|
|
fn main() {
|
|
|
|
|
|
let mut tokenizer_json_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
|
tokenizer_json_path.push("assets");
|
|
tokenizer_json_path.push("tokenizer.json");
|
|
|
|
// 示例：使用 HFTokenizer
|
|
match tokenizers::HFTokenizer::new(tokenizer_json_path) {
|
|
Ok(mut tokenizer) => {
|
|
match tokenizer.gen_predict_sample("hello world", "ni hao") {
|
|
Ok(model_input) => {
|
|
println!("Model input generated successfully");
|
|
println!("Input IDs: {:?}", model_input.input_ids);
|
|
println!("PG value: {}", model_input.pg[[0]]);
|
|
}
|
|
Err(e) => eprintln!("Error generating model input: {}", e),
|
|
}
|
|
}
|
|
Err(e) => eprintln!("Error loading tokenizer: {}", e),
|
|
|
|
}
|
|
}
|