judge/paperless-field-extractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix temperature sampling
Some checks failed
Rust / build (push) Has been cancelled
Details

Browse source 
using temperature requires randomized picking of final token, since the
probability of the most probable token will also be the most probable
after appling temperature, so greedy may not be used!
This commit is contained in:
judge 2025-12-13 15:01:45 +01:00
parent 90d2897dee
commit 3cf2d7fe7c
No known key found for this signature in database
GPG key ID: 6512C30DD8E017B5
3 changed files with 6 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1765,6 +1765,7 @@ dependencies = [
"log", "log",
"once_cell", "once_cell",
"paperless-api-client", "paperless-api-client",
"rand 0.9.2",
"regex", "regex",
"regex_static", "regex_static",
"schemars 1.1.0", "schemars 1.1.0",

1
Cargo.toml
View file

@ -33,6 +33,7 @@ once_cell = "1.21.3"
itertools = "0.14.0" itertools = "0.14.0"
utoipa = { version = "5.4.0", features = ["actix_extras"] } utoipa = { version = "5.4.0", features = ["actix_extras"] }
utoipa-swagger-ui = { version = "9.0.2", features = ["actix-web"] } utoipa-swagger-ui = { version = "9.0.2", features = ["actix-web"] }
rand = "0.9.2"
[features] [features]
vulkan = [ "llama-cpp-2/vulkan" ] vulkan = [ "llama-cpp-2/vulkan" ]

7
src/extract.rs
View file

@ -109,9 +109,10 @@ impl LLModelExtractor {
let grammar = gen_gbnf(response_schema, self.eos_string.to_string()); let grammar = gen_gbnf(response_schema, self.eos_string.to_string());
let mut sampler = LlamaSampler::chain_simple([ let mut sampler = LlamaSampler::chain_simple([
LlamaSampler::grammar(&self.model, &grammar, "root").unwrap(), LlamaSampler::grammar(&self.model, &grammar, "root").unwrap(),
LlamaSampler::dry(&self.model, 5., 1.75, 2, 1024, ["\n", ":", "\"", "*"]), LlamaSampler::dry(&self.model, 5., 1.75, 2, 256, ["\"", ":", "*"], ),
LlamaSampler::temp(0.5), LlamaSampler::min_p(0.01, 64),
LlamaSampler::greedy(), LlamaSampler::temp(0.1),
LlamaSampler::dist(rand::random()),
]); ]);
let prompt = format!("{}\n", serde_json::to_string(base_data).unwrap()); let prompt = format!("{}\n", serde_json::to_string(base_data).unwrap());
let mut ctx = self let mut ctx = self