mod tokenizers;

use std::path::PathBuf;
fn main() {


    let mut tokenizer_json_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    tokenizer_json_path.push("assets");
    tokenizer_json_path.push("tokenizer.json");

    // 示例：使用 HFTokenizer
    match tokenizers::HFTokenizer::new(tokenizer_json_path) {
        Ok(mut tokenizer) => {
            match tokenizer.gen_predict_sample("hello world", "ni hao") {
                Ok(model_input) => {
                    println!("Model input generated successfully");
                    println!("Input IDs: {:?}", model_input.input_ids);
                    println!("PG value: {}", model_input.pg[[0]]);
                }
                Err(e) => eprintln!("Error generating model input: {}", e),
            }
        }
        Err(e) => eprintln!("Error loading tokenizer: {}", e),
 
    }
}