diff --git a/Cargo.lock b/Cargo.lock index 3a0ee9a..5b3894f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1765,6 +1765,7 @@ dependencies = [ "log", "once_cell", "paperless-api-client", + "rand 0.9.2", "regex", "regex_static", "schemars 1.1.0", diff --git a/Cargo.toml b/Cargo.toml index d37b54f..5830445 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ once_cell = "1.21.3" itertools = "0.14.0" utoipa = { version = "5.4.0", features = ["actix_extras"] } utoipa-swagger-ui = { version = "9.0.2", features = ["actix-web"] } +rand = "0.9.2" [features] vulkan = [ "llama-cpp-2/vulkan" ] diff --git a/src/extract.rs b/src/extract.rs index 0be97c9..ae54319 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -109,9 +109,10 @@ impl LLModelExtractor { let grammar = gen_gbnf(response_schema, self.eos_string.to_string()); let mut sampler = LlamaSampler::chain_simple([ LlamaSampler::grammar(&self.model, &grammar, "root").unwrap(), - LlamaSampler::dry(&self.model, 5., 1.75, 2, 1024, ["\n", ":", "\"", "*"]), - LlamaSampler::temp(0.5), - LlamaSampler::greedy(), + LlamaSampler::dry(&self.model, 5., 1.75, 2, 256, ["\"", ":", "*"], ), + LlamaSampler::min_p(0.01, 64), + LlamaSampler::temp(0.1), + LlamaSampler::dist(rand::random()), ]); let prompt = format!("{}\n", serde_json::to_string(base_data).unwrap()); let mut ctx = self