var O=class{constructor(e){this.trie=this._build_trie(e)}_build_trie(e){let t=Object.create(null);for(let r of e){let n=t;for(let s=0;s<r.length;++s){let i=r[s];n=n[i]??=Object.create(null)}n.end=r}return t}split(e){let t=[],r=e.length,n=0,s=0;for(;s<r;){let i=this.trie,c=null,a=s;for(;a<r&&(i=i[e[a]]);)i.end&&(c=i.end),++a;c?(s>n&&t.push(e.slice(n,s)),t.push(c),s+=c.length,n=s):++s}return n<r&&t.push(e.slice(n)),t}},M=O;var R=class{constructor(e){this.content=e.content,this.id=e.id,this.single_word=e.single_word??!1,this.lstrip=e.lstrip??!1,this.rstrip=e.rstrip??!1,this.special=e.special??!1,this.normalized=e.normalized??!this.special}},B=R;var F=(()=>{let o=[...Array.from({length:94},(n,s)=>s+33),...Array.from({length:12},(n,s)=>s+161),...Array.from({length:82},(n,s)=>s+174)],e=o.slice(),t=0;for(let n=0;n<256;++n)o.includes(n)||(o.push(n),e.push(256+t),t+=1);let r=e.map(n=>String.fromCharCode(n));return Object.fromEntries(o.map((n,s)=>[n,r[s]]))})(),qt=o=>Object.fromEntries(Object.entries(o).map(([e,t])=>[t,e])),Ct=qt(F),wt=".,!?\u2026\u3002\uFF0C\u3001\u0964\u06D4\u060C",vt=new Map([["(?i:'s|'t|'re|'ve|'m|'ll|'d)","(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))"],["(?i:[sdmt]|ll|ve|re)","(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])"],["[^\\r\\n\\p{L}\\p{N}]?+","[^\\r\\n\\p{L}\\p{N}]?"],["[^\\s\\p{L}\\p{N}]++","[^\\s\\p{L}\\p{N}]+"],["(?>\\p{Nd}{510})","(?:\\p{Nd}{510})"],["\\p{Nd}{3}+","(?:\\p{Nd}{3})+"],["\\G",""],[` ?[^(\\s|[${wt}])]+`,` ?[^\\s${wt}]+`]]),x="\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";var T=o=>o.replace(/ \./g,".").replace(/ \?/g,"?").replace(/ \!/g,"!").replace(/ ,/g,",").replace(/ \' /g,"'").replace(/ n't/g,"n't").replace(/ 'm/g,"'m").replace(/ 's/g,"'s").replace(/ 've/g,"'ve").replace(/ 're/g,"'re"),z=(o,e=!0)=>{if(o.Regex!==void 0){let t=o.Regex.replace(/\\([#&~])/g,"$1");t=t.replace(/\\A/g,"^").replace(/\\z/g,"$").replace(/\\Z/g,"(?=\\r?\\n?$)");for(let[r,n]of vt)t=t.replaceAll(r,n);try{return new RegExp(t,"gu")}catch(r){if(!(r instanceof SyntaxError)||!r.message.toLowerCase().includes("invalid property name"))throw r;let n=!1,s=t.replace(/(\\[pP])\{([^}=]+)\}/g,(i,c,a)=>{try{return new RegExp(`\\p{${a}}`,"u"),`${c}{${a}}`}catch{return n=!0,`${c}{Script=${a}}`}});if(!n)throw r;try{return new RegExp(s,"gu")}catch{throw r}}}else if(o.String!==void 0){let t=Wt(o.String);return new RegExp(e?t:`(${t})`,"gu")}else return console.warn("Unknown pattern type:",o),null},Wt=o=>o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),Nt=(o,e,t)=>{let r=[],n=0;for(;n<o.length;){if(r.push(o[n]),(e.get(o[n])??t)!==t){++n;continue}for(;++n<o.length&&(e.get(o[n])??t)===t;)e.get(r.at(-1))!==t&&(r[r.length-1]+=o[n])}return r},At=o=>o>=19968&&o<=40959||o>=13312&&o<=19903||o>=131072&&o<=173791||o>=173824&&o<=177983||o>=177984&&o<=178207||o>=178208&&o<=183983||o>=63744&&o<=64255||o>=194560&&o<=195103,Et=o=>Number.isInteger(o)||typeof o=="bigint",Dt=o=>{let e=0;for(let t of o)++e;return e},St=o=>L(o.toLowerCase()),h=(...o)=>Array.prototype.concat.apply([],o),y=o=>new Map(Object.entries(o)),Ot=(o,e)=>{let t=[],r=0;for(let n of o.matchAll(e)){let s=n[0];r<n.index&&t.push(o.slice(r,n.index)),s.length>0&&t.push(s),r=n.index+s.length}return r<o.length&&t.push(o.slice(r)),t},L=o=>o.replace(/\p{M}/gu,""),$=(o,e,t=[])=>{if(!o||Array.isArray(o)||typeof o!="object")return`${e} must be a valid object`;for(let r of t)if(!(r in o))return`${e} must contain a "${r}" property`;return null},Mt=o=>o.match(/\S+/g)||[];var j=class{constructor(){let e=function(...t){return e._call(...t)};return Object.setPrototypeOf(e,new.target.prototype)}},m=j;var U=class extends m{constructor(e){super(),this.config=e}_call(e){return this.normalize(e)}},u=U;var q=class extends u{tokenize_chinese_chars(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r],s=n.charCodeAt(0);At(s)?(t.push(" "),t.push(n),t.push(" ")):t.push(n)}return t.join("")}strip_accents(e){return e.normalize("NFD").replace(/\p{Mn}/gu,"")}is_control(e){switch(e){case"	":case`
`:case"\r":return!1;default:return/^\p{Cc}|\p{Cf}|\p{Co}|\p{Cs}$/u.test(e)}}clean_text(e){let t=[];for(let r of e){let n=r.charCodeAt(0);n===0||n===65533||this.is_control(r)||(/^\s$/.test(r)?t.push(" "):t.push(r))}return t.join("")}normalize(e){return this.config.clean_text&&(e=this.clean_text(e)),this.config.handle_chinese_chars&&(e=this.tokenize_chinese_chars(e)),this.config.lowercase?(e=e.toLowerCase(),this.config.strip_accents!==!1&&(e=this.strip_accents(e))):this.config.strip_accents&&(e=this.strip_accents(e)),e}},W=q;var K=class extends u{constructor(e){super(e),this.charsmap=e.precompiled_charsmap??null}normalize(e){return e=e.replace(/[\u0001-\u0008\u000B\u000E-\u001F\u007F\u008F\u009F]/gm,""),e=e.replace(/[\u0009\u000A\u000C\u000D\u00A0\u1680\u2000-\u200F\u2028\u2029\u202F\u205F\u2581\u3000\uFEFF\uFFFD]/gm," "),e.includes("\uFF5E")?e=e.split("\uFF5E").map(r=>r.normalize("NFKC")).join("\uFF5E"):e=e.normalize("NFKC"),e}},I=K;var G=class extends u{constructor(e){super(e),this.normalizers=(e.normalizers??[]).map(t=>w(t))}normalize(e){return this.normalizers.reduce((t,r)=>r?r.normalize(t):t,e)}},V=G;var J=class extends u{normalize(e){let t=z(this.config.pattern??{});return t===null?e:e.replaceAll(t,this.config.content??"")}},X=J;var Y=class extends u{constructor(){super(...arguments);this.form="NFC"}normalize(t){return t=t.normalize(this.form),t}},b=Y;var H=class extends b{constructor(){super(...arguments);this.form="NFC"}},Q=H;var Z=class extends b{constructor(){super(...arguments);this.form="NFD"}},ee=Z;var te=class extends b{constructor(){super(...arguments);this.form="NFKC"}},re=te;var oe=class extends b{constructor(){super(...arguments);this.form="NFKD"}},ne=oe;var se=class extends u{normalize(e){return this.config.strip_left&&this.config.strip_right?e=e.trim():(this.config.strip_left&&(e=e.trimStart()),this.config.strip_right&&(e=e.trimEnd())),e}},ie=se;var ae=class extends u{normalize(e){return L(e)}},ce=ae;var pe=class extends u{normalize(e){return e.toLowerCase()}},le=pe;var de=class extends u{normalize(e){return e=this.config.prepend+e,e}},ue=de;function Kt(o){if(o===null)return null;switch(o.type){case"BertNormalizer":return new W(o);case"Precompiled":return new I(o);case"Sequence":return new V(o);case"Replace":return new X(o);case"NFC":return new Q(o);case"NFD":return new ee(o);case"NFKC":return new re(o);case"NFKD":return new ne(o);case"Strip":return new ie(o);case"StripAccents":return new ce(o);case"Lowercase":return new le(o);case"Prepend":return new ue(o);default:throw new Error(`Unknown Normalizer type: ${o.type}`)}}var w=Kt;var fe=class extends m{pre_tokenize(e,t){return(Array.isArray(e)?e.map(r=>this.pre_tokenize_text(r,t)):this.pre_tokenize_text(e,t)).flat()}_call(e,t){return this.pre_tokenize(e,t)}},l=fe;var he=class extends l{constructor(e){super(),this.config=e,this.add_prefix_space=this.config.add_prefix_space??!1,this.trim_offsets=this.config.trim_offsets??!1,this.use_regex=this.config.use_regex??!0,this.pattern=/'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu,this.byte_encoder=F,this.text_encoder=new TextEncoder}pre_tokenize_text(e,t){return this.add_prefix_space&&!e.startsWith(" ")&&(e=" "+e),(this.use_regex?e.match(this.pattern)||[]:[e]).map(n=>Array.from(this.text_encoder.encode(n),s=>this.byte_encoder[s]).join(""))}},me=he;var _e=class extends l{pre_tokenize_text(e,t){return e.match(/\w+|[^\w\s]+/g)||[]}},ge=_e;var ke=class extends l{constructor(e){super(),this.replacement=e.replacement??"\u2581",this.str_rep=e.str_rep||this.replacement,this.prepend_scheme=e.prepend_scheme??"always"}pre_tokenize_text(e,t){let{section_index:r=void 0}=t??{},n=e.replaceAll(" ",this.str_rep);return!n.startsWith(this.replacement)&&(this.prepend_scheme==="always"||this.prepend_scheme==="first"&&r===0)&&(n=this.str_rep+n),[n]}},ze=ke;var be=class extends l{constructor(e){super(),this.config=e,this.pattern=z(this.config.pattern??{},this.config.invert??!0)}pre_tokenize_text(e){return this.pattern===null?[]:this.config.invert?e.match(this.pattern)||[]:this.config.behavior?.toLowerCase()==="removed"?e.split(this.pattern).filter(t=>t):Ot(e,this.pattern)}},xe=be;var Te=class extends l{constructor(e){super(),this.config=e,this.pattern=new RegExp(`[^${x}]+|[${x}]+`,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},ye=Te;var Pe=class extends l{constructor(e){super(),this.config=e;let t=`[^\\d]+|\\d${this.config.individual_digits?"":"+"}`;this.pattern=new RegExp(t,"gu")}pre_tokenize_text(e){return e.match(this.pattern)||[]}},we=Pe;var Ce=class extends l{constructor(){super(),this.pattern=new RegExp(`[^\\s${x}]+|[${x}]`,"gu")}pre_tokenize_text(e,t){return e.trim().match(this.pattern)||[]}},ve=Ce;var Ne=class extends l{constructor(e){super(),this.config=e,this.pattern=z(this.config.pattern??{}),this.content=this.config.content??""}pre_tokenize_text(e){return this.pattern===null?[e]:[e.replaceAll(this.pattern,this.config.content??"")]}},Ae=Ne;var Ee=class extends l{constructor(e){super(),this.tokenizers=(e.pretokenizers??[]).map(t=>C(t))}pre_tokenize_text(e,t){return this.tokenizers.reduce((r,n)=>n?n.pre_tokenize(r,t):r,[e])}},De=Ee;var Se=class extends l{pre_tokenize_text(e){return Mt(e)}},Oe=Se;var Me=class extends l{constructor(e){super(),this.config=e,this._length=e.length}pre_tokenize_text(e){let t=[];for(let r=0;r<e.length;r+=this._length)t.push(e.slice(r,r+this._length));return t}},Re=Me;function It(o){if(o===null)return null;switch(o.type){case"BertPreTokenizer":return new ve;case"Sequence":return new De(o);case"Whitespace":return new ge;case"WhitespaceSplit":return new Oe;case"Metaspace":return new ze(o);case"ByteLevel":return new me(o);case"Split":return new xe(o);case"Punctuation":return new ye(o);case"Digits":return new we(o);case"Replace":return new Ae(o);case"FixedLength":return new Re(o);default:throw new Error(`Unknown PreTokenizer type: ${o.type}`)}}var C=It;var Be=class extends m{constructor(e){super(),this.config=e,this.vocab=[],this.tokens_to_ids=new Map,this.unk_token_id=void 0,this.unk_token=void 0,this.end_of_word_suffix=void 0,this.fuse_unk=this.config.fuse_unk??!1}_call(e){let t=this.encode(e);return this.fuse_unk&&(t=Nt(t,this.tokens_to_ids,this.unk_token_id)),t}},k=Be;var Fe=class extends k{constructor(t){super(t);this.max_input_chars_per_word=100;this.tokens_to_ids=y(t.vocab),this.unk_token_id=this.tokens_to_ids.get(t.unk_token),this.unk_token=t.unk_token,this.max_input_chars_per_word=t.max_input_chars_per_word??100,this.vocab=new Array(this.tokens_to_ids.size);for(let[r,n]of this.tokens_to_ids)this.vocab[n]=r}encode(t){let r=[];for(let n of t){let s=[...n];if(s.length>this.max_input_chars_per_word){r.push(this.unk_token);continue}let i=!1,c=0,a=[];for(;c<s.length;){let p=s.length,f=null;for(;c<p;){let g=s.slice(c,p).join("");if(c>0&&(g=this.config.continuing_subword_prefix+g),this.tokens_to_ids.has(g)){f=g;break}--p}if(f===null){i=!0;break}a.push(f),c=p}i?r.push(this.unk_token):r.push(...a)}return r}},v=Fe;var N=class o{constructor(e,t){this.is_leaf=e,this.children=t}static default(){return new o(!1,new Map)}},Le=class{constructor(){this.root=N.default()}extend(e){for(let t of e)this.push(t)}push(e){let t=this.root;for(let r of e){let n=t.children.get(r);n===void 0&&(n=N.default(),t.children.set(r,n)),t=n}t.is_leaf=!0}*common_prefix_search(e){let t=this.root;if(t===void 0)return;let r="";for(let n of e){if(r+=n,t=t.children.get(n),t===void 0)return;t.is_leaf&&(yield r)}}},Rt=Le;var P=class o{constructor(e,t,r,n,s){this.token_id=e,this.node_id=t,this.pos=r,this.length=n,this.score=s,this.prev=null,this.backtrace_score=0}clone(){let e=new o(this.token_id,this.node_id,this.pos,this.length,this.score);return e.prev=this.prev,e.backtrace_score=this.backtrace_score,e}},$e=class{constructor(e,t,r){this.chars=Array.from(e),this.len=this.chars.length,this.bos_token_id=t,this.eos_token_id=r,this.nodes=[],this.begin_nodes=Array.from({length:this.len+1},()=>[]),this.end_nodes=Array.from({length:this.len+1},()=>[]);let n=new P(this.bos_token_id??0,0,0,0,0),s=new P(this.eos_token_id??0,1,this.len,0,0);this.nodes.push(n.clone()),this.nodes.push(s.clone()),this.begin_nodes[this.len].push(s),this.end_nodes[0].push(n)}insert(e,t,r,n){let s=this.nodes.length,i=new P(n,s,e,t,r);this.begin_nodes[e].push(i),this.end_nodes[e+t].push(i),this.nodes.push(i)}viterbi(){let e=this.len,t=0;for(;t<=e;){if(this.begin_nodes[t].length==0)return[];for(let c of this.begin_nodes[t]){c.prev=null;let a=0,p=null;for(let f of this.end_nodes[t]){let g=f.backtrace_score+c.score;(p===null||g>a)&&(p=f.clone(),a=g)}if(p!==null)c.prev=p,c.backtrace_score=a;else return[]}++t}let r=[],s=this.begin_nodes[e][0].prev;if(s===null)return[];let i=s.clone();for(;i.prev!==null;)r.push(i.clone()),i=i.clone().prev.clone();return r.reverse(),r}piece(e){return this.chars.slice(e.pos,e.pos+e.length).join("")}tokens(){return this.viterbi().map(t=>this.piece(t))}token_ids(){return this.viterbi().map(t=>t.token_id)}},Bt=$e;function Ft(o){if(o.length===0)throw new Error("Array must not be empty");let e=o[0],t=0;for(let r=1;r<o.length;++r)o[r]<e&&(e=o[r],t=r);return[e,t]}var je=class extends k{constructor(e,t){super(e);let r=e.vocab.length;this.vocab=new Array(r),this.scores=new Array(r);for(let n=0;n<r;++n)[this.vocab[n],this.scores[n]]=e.vocab[n];this.unk_token_id=e.unk_id,this.unk_token=this.vocab[e.unk_id],this.tokens_to_ids=new Map(this.vocab.map((n,s)=>[n,s])),this.bos_token=" ",this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.unk_token=this.vocab[this.unk_token_id],this.min_score=Ft(this.scores)[0],this.unk_score=this.min_score-10,this.scores[this.unk_token_id]=this.unk_score,this.trie=new Rt,this.trie.extend(this.vocab),this.fuse_unk=!0}populate_nodes(e){let t=e.chars,r=1,n=0;for(;n<t.length;){let s=!1,i=[],c=t.slice(n).join(""),a=this.trie.common_prefix_search(c);for(let p of a){i.push(p);let f=this.tokens_to_ids.get(p),g=this.scores[f],Pt=Dt(p);e.insert(n,Pt,g,f),!s&&Pt===r&&(s=!0)}s||e.insert(n,r,this.unk_score,this.unk_token_id),n+=r}}tokenize(e){let t=new Bt(e,this.bos_token_id,this.eos_token_id);return this.populate_nodes(t),t.tokens()}encode(e){let t=[];for(let r of e){let n=this.tokenize(r);t.push(...n)}return t}},A=je;var Ue=class{constructor(e=(r,n)=>r>n,t=1/0){this._heap=[],this._comparator=e,this._max_size=t}get size(){return this._heap.length}is_empty(){return this.size===0}peek(){return this._heap[0]}push(...e){return this.extend(e)}extend(e){for(let t of e)if(this.size<this._max_size)this._heap.push(t),this._sift_up();else{let r=this._smallest();this._comparator(t,this._heap[r])&&(this._heap[r]=t,this._sift_up_from(r))}return this.size}pop(){let e=this.peek(),t=this.size-1;return t>0&&this._swap(0,t),this._heap.pop(),this._sift_down(),e}replace(e){let t=this.peek();return this._heap[0]=e,this._sift_down(),t}_parent(e){return(e+1>>>1)-1}_left(e){return(e<<1)+1}_right(e){return e+1<<1}_greater(e,t){return this._comparator(this._heap[e],this._heap[t])}_swap(e,t){let r=this._heap[e];this._heap[e]=this._heap[t],this._heap[t]=r}_sift_up(){this._sift_up_from(this.size-1)}_sift_up_from(e){for(;e>0&&this._greater(e,this._parent(e));)this._swap(e,this._parent(e)),e=this._parent(e)}_sift_down(){let e=0;for(;this._left(e)<this.size&&this._greater(this._left(e),e)||this._right(e)<this.size&&this._greater(this._right(e),e);){let t=this._right(e)<this.size&&this._greater(this._right(e),this._left(e))?this._right(e):this._left(e);this._swap(e,t),e=t}}_smallest(){return 2**Math.floor(Math.log2(this.size))-1}},Lt=Ue;var qe=class{constructor(e){this.capacity=e,this.cache=new Map}get(e){if(!this.cache.has(e))return;let t=this.cache.get(e);return this.cache.delete(e),this.cache.set(e,t),t}put(e,t){this.cache.has(e)&&this.cache.delete(e),this.cache.set(e,t),this.cache.size>this.capacity&&this.cache.delete(this.cache.keys().next().value)}clear(){this.cache.clear()}},$t=qe;var We=class extends k{constructor(e){super(e),this.tokens_to_ids=y(e.vocab),this.unk_token_id=this.tokens_to_ids.get(e.unk_token),this.unk_token=e.unk_token,this.vocab=new Array(this.tokens_to_ids.size);for(let[r,n]of this.tokens_to_ids)this.vocab[n]=r;let t=Array.isArray(e.merges[0]);this.merges=t?e.merges:e.merges.map(r=>r.split(" ",2)),this.bpe_ranks=new Map(this.merges.map((r,n)=>[JSON.stringify(r),n])),this.end_of_word_suffix=e.end_of_word_suffix,this.continuing_subword_suffix=e.continuing_subword_suffix??null,this.byte_fallback=this.config.byte_fallback??!1,this.byte_fallback&&(this.text_encoder=new TextEncoder),this.ignore_merges=this.config.ignore_merges??!1,this.max_length_to_cache=256,this.cache_capacity=1e4,this.cache=new $t(this.cache_capacity)}clear_cache(){this.cache.clear()}bpe(e){if(e.length===0)return[];let t=this.cache.get(e);if(t!==void 0)return t;let r=Array.from(e);this.end_of_word_suffix&&(r[r.length-1]+=this.end_of_word_suffix);let n=[];if(r.length>1){let s=new Lt((a,p)=>a.score<p.score),i={token:r[0],bias:0,prev:null,next:null},c=i;for(let a=1;a<r.length;++a){let p={bias:a/r.length,token:r[a],prev:c,next:null};c.next=p,this.add_node(s,c),c=p}for(;!s.is_empty();){let a=s.pop();if(a.deleted||!a.next||a.next.deleted)continue;if(a.deleted=!0,a.next.deleted=!0,a.prev){let f={...a.prev};a.prev.deleted=!0,a.prev=f,f.prev?f.prev.next=f:i=f}let p={token:a.token+a.next.token,bias:a.bias,prev:a.prev,next:a.next.next};p.prev?(p.prev.next=p,this.add_node(s,p.prev)):i=p,p.next&&(p.next.prev=p,this.add_node(s,p))}for(let a=i;a!==null;a=a.next)n.push(a.token)}else n=r;if(this.continuing_subword_suffix)for(let s=0;s<n.length-1;++s)n[s]+=this.continuing_subword_suffix;return e.length<this.max_length_to_cache&&this.cache.put(e,n),n}add_node(e,t){let r=this.bpe_ranks.get(JSON.stringify([t.token,t.next.token]));r!==void 0&&(t.score=r+t.bias,e.push(t))}encode(e){let t=[];for(let r of e){if(this.ignore_merges&&this.tokens_to_ids.has(r)){t.push(r);continue}let n=this.bpe(r);for(let s of n)if(this.tokens_to_ids.has(s))t.push(s);else if(this.byte_fallback){let i=Array.from(this.text_encoder.encode(s)).map(c=>`<0x${c.toString(16).toUpperCase().padStart(2,"0")}>`);i.every(c=>this.tokens_to_ids.has(c))?t.push(...i):this.unk_token!=null&&t.push(this.unk_token)}else this.unk_token!=null&&t.push(this.unk_token)}return t}},E=We;var Ke=class extends k{constructor(e,t){super(e);let r=e.vocab;this.tokens_to_ids=y(t.target_lang?r[t.target_lang]:r),this.bos_token=t.bos_token,this.bos_token_id=this.tokens_to_ids.get(this.bos_token),this.eos_token=t.eos_token,this.eos_token_id=this.tokens_to_ids.get(this.eos_token),this.pad_token=t.pad_token,this.pad_token_id=this.tokens_to_ids.get(this.pad_token),this.unk_token=t.unk_token,this.unk_token_id=this.tokens_to_ids.get(this.unk_token),this.vocab=new Array(this.tokens_to_ids.size);for(let[n,s]of this.tokens_to_ids)this.vocab[s]=n}encode(e){return e}},jt=Ke;function Gt(o,e){switch(o.type){case"WordPiece":return new v(o);case"Unigram":return new A(o,e.eos_token);case"BPE":return new E(o);default:if(o.vocab)return Array.isArray(o.vocab)?new A(o,e.eos_token):Object.hasOwn(o,"continuing_subword_prefix")&&Object.hasOwn(o,"unk_token")?Object.hasOwn(o,"merges")?new E(o):new v(o):new jt(o,{target_lang:e.target_lang,bos_token:e.bos_token,eos_token:e.eos_token,pad_token:e.pad_token,unk_token:e.unk_token});throw new Error(`Unknown TokenizerModel type: ${o?.type}`)}}var Ut=Gt;var Ie=class extends m{constructor(e){super(),this.config=e}_call(e,...t){return this.post_process(e,...t)}},_=Ie;var Ge=class extends _{post_process(e,t=null,r=!0){let n=t===null?this.config.single:this.config.pair,s=[],i=[];for(let c of n)"SpecialToken"in c?r&&(s.push(c.SpecialToken.id),i.push(c.SpecialToken.type_id)):"Sequence"in c&&(c.Sequence.id==="A"?(s=h(s,e),i=h(i,new Array(e.length).fill(c.Sequence.type_id))):c.Sequence.id==="B"&&(s=h(s,t),i=h(i,new Array(t.length).fill(c.Sequence.type_id))));return{tokens:s,token_type_ids:i}}},Ve=Ge;var Je=class extends _{post_process(e,t=null){return{tokens:e,tokens_pair:t}}},Xe=Je;var Ye=class extends _{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t=null,r=!0){r&&(e=h([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let s=[],i=r?[this.sep[0]]:[];e=h(e,s,t,i),n=h(n,new Array(t.length+s.length+i.length).fill(1))}return{tokens:e,token_type_ids:n}}},He=Ye;var Qe=class extends _{constructor(e){super(e),this.sep=e.sep,this.cls=e.cls}post_process(e,t,r=!0){r&&(e=h([this.cls[0]],e,[this.sep[0]]));let n=new Array(e.length).fill(0);if(t){let s=r?[this.sep[0]]:[],i=r?[this.sep[0]]:[];e=h(e,s,t,i),n=h(n,new Array(t.length+s.length+i.length).fill(1))}return{tokens:e,token_type_ids:n}}},Ze=Qe;var et=class extends _{constructor(e){super(e),this.processors=(e.processors??[]).map(t=>D(t))}post_process(e,t=null,r=!0){let n={tokens:e,tokens_pair:t};for(let s of this.processors)n=s.post_process(n.tokens,n.tokens_pair,r);return n}},tt=et;function Vt(o){if(o===null)return null;switch(o.type){case"TemplateProcessing":return new Ve(o);case"ByteLevel":return new Xe(o);case"BertProcessing":return new He(o);case"RobertaProcessing":return new Ze(o);case"Sequence":return new tt(o);default:throw new Error(`Unknown PostProcessor type: ${o.type}`)}}var D=Vt;var rt=class extends m{constructor(e){super(),this.config=e,this.added_tokens=[],this.end_of_word_suffix=null,this.trim_offsets="trim_offsets"in e?e.trim_offsets:!1}_call(e){return this.decode(e)}decode(e){return this.decode_chain(e).join("")}},d=rt;var ot=class extends d{constructor(e){super(e),this.byte_decoder=Ct,this.text_decoder=new TextDecoder("utf-8",{fatal:!1,ignoreBOM:!0}),this.end_of_word_suffix=null}convert_tokens_to_string(e){let t=e.join(""),r=new Uint8Array([...t].map(n=>this.byte_decoder[n]));return this.text_decoder.decode(r)}decode_chain(e){let t=[],r=[];for(let n of e)this.added_tokens.find(s=>s.content===n)!==void 0?(r.length>0&&(t.push(this.convert_tokens_to_string(r)),r=[]),t.push(n)):r.push(n);return r.length>0&&t.push(this.convert_tokens_to_string(r)),t}},nt=ot;var st=class extends d{constructor(e){super(e),this.cleanup=e.cleanup}decode_chain(e){return e.map((t,r)=>{if(r!==0){let n=this.config.prefix;n&&t.startsWith(n)?t=t.replace(n,""):t=" "+t}return this.cleanup&&(t=T(t)),t})}},it=st;var at=class extends d{constructor(e){super(e),this.replacement=e.replacement??"\u2581"}decode_chain(e){let t=[];for(let r=0;r<e.length;++r){let n=e[r].replaceAll(this.replacement," ");r==0&&n.startsWith(" ")&&(n=n.substring(1)),t.push(n)}return t}},ct=at;var pt=class extends d{constructor(e){super(e),this.suffix=e.suffix??""}decode_chain(e){return e.map((t,r)=>t.replaceAll(this.suffix,r===e.length-1?"":" "))}},lt=pt;var dt=class extends d{constructor(e){super(e),this.pad_token=e.pad_token??"",this.word_delimiter_token=e.word_delimiter_token??"",this.cleanup=e.cleanup}convert_tokens_to_string(e){if(e.length===0)return"";let t=[e[0]];for(let s=1;s<e.length;++s)e[s]!==t.at(-1)&&t.push(e[s]);let n=t.filter(s=>s!==this.pad_token).join("");return this.cleanup&&(n=T(n).replaceAll(this.word_delimiter_token," ").trim()),n}decode_chain(e){return[this.convert_tokens_to_string(e)]}},ut=dt;var ft=class extends d{constructor(e){super(e),this.decoders=(e.decoders??[]).map(t=>S(t))}decode_chain(e){return this.decoders.reduce((t,r)=>r.decode_chain(t),e)}},ht=ft;var mt=class extends d{decode_chain(e){let t=z(this.config.pattern),r=this.config.content??"";return t===null?e:e.map(n=>n.replaceAll(t,r))}},_t=mt;var gt=class extends d{decode_chain(e){return[e.join("")]}},kt=gt;var zt=class extends d{constructor(e){super(e),this.content=e.content??"",this.start=e.start??0,this.stop=e.stop??0}decode_chain(e){return e.map(t=>{let r=0;for(let s=0;s<this.start&&t[s]===this.content;++s){r=s+1;continue}let n=t.length;for(let s=0;s<this.stop;++s){let i=t.length-s-1;if(t[i]===this.content){n=i;continue}else break}return t.slice(r,n)})}},bt=zt;var xt=class extends d{constructor(e){super(e),this.text_decoder=new TextDecoder}decode_chain(e){let t=[],r=[];for(let n of e){let s=null;if(n.length===6&&n.startsWith("<0x")&&n.endsWith(">")){let i=parseInt(n.slice(3,5),16);isNaN(i)||(s=i)}if(s!==null)r.push(s);else{if(r.length>0){let i=this.text_decoder.decode(Uint8Array.from(r));t.push(i),r=[]}t.push(n)}}if(r.length>0){let n=this.text_decoder.decode(Uint8Array.from(r));t.push(n),r=[]}return t}},Tt=xt;function Jt(o){if(o===null)return null;switch(o.type){case"ByteLevel":return new nt(o);case"WordPiece":return new it(o);case"Metaspace":return new ct(o);case"BPEDecoder":return new lt(o);case"CTC":return new ut(o);case"Sequence":return new ht(o);case"Replace":return new _t(o);case"Fuse":return new kt(o);case"Strip":return new bt(o);case"ByteFallback":return new Tt(o);default:throw new Error(`Unknown Decoder type: ${o.type}`)}}var S=Jt;var yt=class{constructor(e,t){let r=$(e,"Tokenizer",["model","decoder","post_processor","pre_tokenizer","normalizer"]);if(r)throw new Error(r);let n=$(t,"Config");if(n)throw new Error(n);this.tokenizer=e,this.config=t,this.normalizer=w(this.tokenizer.normalizer),this.pre_tokenizer=C(this.tokenizer.pre_tokenizer),this.model=Ut(this.tokenizer.model,this.config),this.post_processor=D(this.tokenizer.post_processor),this.decoder=S(this.tokenizer.decoder),this.special_tokens=[],this.all_special_ids=[],this.added_tokens=[];let s=[],i=[];this.added_tokens_map=new Map;for(let c of this.tokenizer.added_tokens){let a=new B(c);if(this.added_tokens.push(a),this.model.tokens_to_ids.set(a.content,a.id),this.model.vocab[a.id]=a.content,a.special&&(this.special_tokens.push(a.content),this.all_special_ids.push(a.id)),this.added_tokens_map.set(a.content,a),a.normalized&&this.normalizer!==null){let p=this.normalizer(a.content);i.push(p),this.added_tokens_map.set(p,a)}else s.push(a.content)}(this.config.additional_special_tokens??[]).forEach(c=>{this.special_tokens.includes(c)||this.special_tokens.push(c)}),this.decoder&&(this.decoder.added_tokens=this.added_tokens,this.decoder.end_of_word_suffix=this.model.end_of_word_suffix),this.splitter_unnormalized=new M(s),this.splitter_normalized=new M(i),this.remove_space=this.config.remove_space,this.clean_up_tokenization_spaces=this.config.clean_up_tokenization_spaces??!0,this.do_lowercase_and_remove_accent=this.config.do_lowercase_and_remove_accent??!1}encode(e,{text_pair:t=null,add_special_tokens:r=!0,return_token_type_ids:n=null}={}){let{tokens:s,token_type_ids:i}=this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}),c=s.map(p=>this.added_tokens_map.get(p)?.id??this.model.tokens_to_ids.get(p)??this.model.unk_token_id),a={ids:c,tokens:s,attention_mask:new Array(c.length).fill(1)};return n&&i&&(a.token_type_ids=i),a}decode(e,t={}){if(!Array.isArray(e)||e.length===0||!Et(e[0]))throw Error("token_ids must be a non-empty array of integers.");let r=e.map(s=>this.model.vocab[Number(s)]??this.model.unk_token);t.skip_special_tokens&&(r=r.filter(s=>!this.special_tokens.includes(s)));let n=this.decoder?this.decoder(r):r.join(" ");return this.decoder&&this.decoder.end_of_word_suffix&&(n=n.replaceAll(this.decoder.end_of_word_suffix," "),t.skip_special_tokens&&(n=n.trim())),(t.clean_up_tokenization_spaces??this.clean_up_tokenization_spaces)&&(n=T(n)),n}tokenize(e,{text_pair:t=null,add_special_tokens:r=!1}={}){return this.tokenize_helper(e,{text_pair:t,add_special_tokens:r}).tokens}encode_text(e){if(e===null)return null;let t=this.splitter_unnormalized.split(e);return t.forEach((r,n)=>{let s=this.added_tokens_map.get(r);s&&(s.lstrip&&n>0&&(t[n-1]=t[n-1].trimEnd()),s.rstrip&&n<t.length-1&&(t[n+1]=t[n+1].trimStart()))}),t.flatMap((r,n)=>{if(r.length===0)return[];if(this.added_tokens_map.has(r))return[r];if(this.remove_space===!0&&(r=r.trim().split(/\s+/).join(" ")),this.do_lowercase_and_remove_accent&&(r=St(r)),this.normalizer!==null&&(r=this.normalizer(r)),r.length===0)return[];let s=this.splitter_normalized.split(r);return s.forEach((i,c)=>{let a=this.added_tokens_map.get(i);a&&(a.lstrip&&c>0&&(s[c-1]=s[c-1].trimEnd()),a.rstrip&&c<s.length-1&&(s[c+1]=s[c+1].trimStart()))}),s.flatMap(i=>{if(i.length===0)return[];if(this.added_tokens_map.has(i))return[i];let c=this.pre_tokenizer!==null?this.pre_tokenizer(i,{section_index:n}):[i];return this.model(c)})})}tokenize_helper(e,{text_pair:t=null,add_special_tokens:r=!0}){let n=this.encode_text(e),s=this.encode_text(t||null);return this.post_processor?this.post_processor(n,s,r):{tokens:h(n??[],s??[])}}token_to_id(e){return this.model.tokens_to_ids.get(e)}id_to_token(e){return this.model.vocab[e]}get_added_tokens_decoder(){let e=new Map;for(let t of this.added_tokens)e.set(t.id,t);return e}get_vocab(e=!0){let t=new Map;for(let r=0;r<this.model.vocab.length;++r){let n=this.model.vocab[r];(e||!this.added_tokens_map.has(n))&&t.set(n,r)}return t}},Xt=yt;export{B as AddedToken,E as BPE,lt as BPEDecoder,W as BertNormalizer,ve as BertPreTokenizer,He as BertProcessingPostProcessor,Tt as ByteFallbackDecoder,nt as ByteLevelDecoder,Xe as ByteLevelPostProcessor,me as ByteLevelPreTokenizer,ut as CTCDecoder,d as Decoder,we as DigitsPreTokenizer,Re as FixedLengthPreTokenizer,kt as FuseDecoder,le as LowercaseNormalizer,ct as MetaspaceDecoder,ze as MetaspacePreTokenizer,k as Model,Q as NFCNormalizer,ee as NFDNormalizer,re as NFKCNormalizer,ne as NFKDNormalizer,u as Normalizer,_ as PostProcessor,l as PreTokenizer,I as PrecompiledNormalizer,ue as PrependNormalizer,ye as PunctuationPreTokenizer,_t as ReplaceDecoder,X as ReplaceNormalizer,Ae as ReplacePreTokenizer,Ze as RobertaProcessingPostProcessor,ht as SequenceDecoder,V as SequenceNormalizer,tt as SequencePostProcessor,De as SequencePreTokenizer,xe as SplitPreTokenizer,ce as StripAccentsNormalizer,bt as StripDecoder,ie as StripNormalizer,Ve as TemplateProcessingPostProcessor,Xt as Tokenizer,A as Unigram,ge as WhitespacePreTokenizer,Oe as WhitespaceSplitPreTokenizer,v as WordPiece,it as WordPieceDecoder};