Token、Embedding 與上下文窗（實驗場）

import%20marimo%0A%0A__generated_with%20%3D%20%220.23.16%22%0Aapp%20%3D%20marimo.App(width%3D%22medium%22%2C%20app_title%3D%22Token%E3%80%81Embedding%20%E8%88%87%E4%B8%8A%E4%B8%8B%E6%96%87%E7%AA%97%EF%BC%88%E5%AF%A6%E9%A9%97%E5%A0%B4%EF%BC%89%22)%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%23%20%F0%9F%A7%AA%20Token%E3%80%81Embedding%20%E8%88%87%E4%B8%8A%E4%B8%8B%E6%96%87%E7%AA%97%EF%BC%88%E5%AF%A6%E9%A9%97%E5%A0%B4%EF%BC%89%0A%0A%20%20%20%20%E9%80%99%E6%98%AF%E6%9C%AC%E8%AA%B2%E7%9A%84**%E5%AF%A6%E9%A9%97%E5%A0%B4**%E3%80%82%E5%B7%A6%E5%81%B4%E6%95%99%E5%AD%B8%E8%AE%80%E5%88%B0%E5%93%AA%EF%BC%8C%E5%B0%B1%E5%9B%9E%E5%88%B0%E9%80%99%E8%A3%A1%E5%8B%95%E6%89%8B%E5%81%9A%E2%80%94%E2%80%94%0A%20%20%20%20%E6%AF%8F%E4%B8%80%E6%A0%BC%E7%A8%8B%E5%BC%8F%E7%A2%BC%E9%83%BD%E5%8F%AF%E4%BB%A5**%E7%9B%B4%E6%8E%A5%E4%BF%AE%E6%94%B9%E3%80%81%E7%AB%8B%E5%8D%B3%E9%87%8D%E8%B7%91**%EF%BC%88%E9%BB%9E%E6%A0%BC%E5%AD%90%E5%8F%B3%E4%B8%8A%E7%9A%84%20%E2%96%B6%EF%BC%8C%E6%88%96%E6%8C%89%20%60Ctrl%2BEnter%60%EF%BC%89%E3%80%82%0A%20%20%20%20%E6%94%B9%E5%A3%9E%E4%BA%86%E4%B9%9F%E6%B2%92%E9%97%9C%E4%BF%82%EF%BC%9A%E9%87%8D%E6%96%B0%E6%95%B4%E7%90%86%E9%A0%81%E9%9D%A2%E5%B0%B1%E6%9C%83%E5%9B%9E%E5%88%B0%E5%8E%9F%E7%89%88%E3%80%82%0A%0A%20%20%20%20%E9%80%99%E4%B8%80%E8%AA%B2%E8%A6%81%E8%A6%AA%E6%89%8B%E6%91%B8%E5%9B%9B%E5%80%8B%E5%9C%B0%E5%9F%BA%E5%90%8D%E8%A9%9E%EF%BC%9A**Token**%EF%BC%88%E6%96%87%E5%AD%97%E6%80%8E%E9%BA%BC%E8%A2%AB%E5%88%87%E9%96%8B%EF%BC%89%E3%80%81**Embedding**%EF%BC%88%E8%AA%9E%E6%84%8F%E6%80%8E%E9%BA%BC%E8%AE%8A%E6%88%90%E5%BA%A7%E6%A8%99%EF%BC%89%E3%80%81%0A%20%20%20%20**Context%20Window**%EF%BC%88%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AC%A1%E8%83%BD%E7%9C%8B%E5%A4%9A%E5%B0%91%EF%BC%89%E3%80%81**Autoregressive**%EF%BC%88%E7%82%BA%E4%BB%80%E9%BA%BC%E5%9B%9E%E7%AD%94%E6%98%AF%E4%B8%80%E5%80%8B%E5%AD%97%E4%B8%80%E5%80%8B%E5%AD%97%E8%B9%A6%E5%87%BA%E4%BE%86%EF%BC%89%E3%80%82%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_()%3A%0A%20%20%20%20import%20marimo%20as%20mo%0A%0A%20%20%20%20return%20(mo%2C)%0A%0A%0A%40app.cell%0Adef%20_()%3A%0A%20%20%20%20%23%20%E7%A7%91%E5%AD%B8%E5%A5%97%E4%BB%B6%E9%9B%86%E4%B8%AD%E5%9C%A8%E9%80%99%E6%A0%BC%20import%EF%BC%8C%E5%9B%9E%E5%82%B3%E7%B5%A6%E5%85%A8%20notebook%20%E7%94%A8%0A%20%20%20%20import%20html%20as%20html_mod%0A%20%20%20%20from%20itertools%20import%20pairwise%0A%0A%20%20%20%20import%20matplotlib.pyplot%20as%20plt%0A%20%20%20%20import%20numpy%20as%20np%0A%0A%20%20%20%20return%20html_mod%2C%20np%2C%20pairwise%2C%20plt%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%23%23%201%EF%B8%8F%E2%83%A3%20Token%EF%BC%9A%E6%96%87%E5%AD%97%E6%98%AF%E6%80%8E%E9%BA%BC%E8%A2%AB%E5%88%87%E9%96%8B%E7%9A%84%0A%0A%20%20%20%20%E6%A8%A1%E5%9E%8B%E7%9C%8B%E4%B8%8D%E6%87%82%E3%80%8C%E5%AD%97%E3%80%8D%EF%BC%8C%E5%8F%AA%E7%9C%8B%E5%BE%97%E6%87%82**%E7%B7%A8%E8%99%9F**%E3%80%82Tokenizer%20%E7%9A%84%E5%B7%A5%E4%BD%9C%E5%B0%B1%E6%98%AF%E6%8A%8A%E6%96%87%E5%AD%97%E5%88%87%E6%88%90%E4%B8%80%E5%B0%8F%E5%A1%8A%E4%B8%80%E5%B0%8F%E5%A1%8A%0A%20%20%20%20%EF%BC%88token%EF%BC%89%EF%BC%8C%E6%AF%8F%E5%A1%8A%E7%B5%A6%E4%B8%80%E5%80%8B%E7%B7%A8%E8%99%9F%E3%80%82%E4%B8%BB%E6%B5%81%E7%9A%84%E5%88%87%E6%B3%95%E5%8F%AB%20**BPE%EF%BC%88Byte%20Pair%20Encoding%EF%BC%89**%EF%BC%8C%0A%20%20%20%20%E8%A6%8F%E5%89%87%E7%B0%A1%E5%96%AE%E5%88%B0%E4%BD%A0%E5%8F%AF%E4%BB%A5%E5%9C%A8%E4%B8%8B%E9%9D%A2%E8%A6%AA%E6%89%8B%E8%A8%93%E7%B7%B4%E4%B8%80%E5%80%8B%EF%BC%9A%0A%0A%20%20%20%201.%20%E4%B8%80%E9%96%8B%E5%A7%8B%EF%BC%8C%E6%AF%8F%E5%80%8B%E5%AD%97%E5%85%83%E8%87%AA%E5%B7%B1%E6%98%AF%E4%B8%80%E5%A1%8A%0A%20%20%20%202.%20%E7%B5%B1%E8%A8%88%E8%AA%9E%E6%96%99%E8%A3%A1%E3%80%8C%E5%93%AA%E5%85%A9%E5%A1%8A%E6%9C%80%E5%B8%B8%E7%9B%B8%E9%84%B0%E5%87%BA%E7%8F%BE%E3%80%8D%EF%BC%8C%E6%8A%8A%E5%AE%83%E5%80%91**%E9%BB%8F%E6%88%90%E4%B8%80%E5%A1%8A**%0A%20%20%20%203.%20%E9%87%8D%E8%A4%87%E7%AC%AC%202%20%E6%AD%A5%E5%B9%BE%E5%8D%83%E6%AC%A1%E2%80%94%E2%80%94%E5%B8%B8%E8%A6%8B%E7%9A%84%E5%AD%97%E4%B8%B2%E5%B0%B1%E8%AE%8A%E6%88%90%E4%B8%80%E6%95%B4%E5%A1%8A%EF%BC%8C%E7%BD%95%E8%A6%8B%E7%9A%84%E7%B6%AD%E6%8C%81%E5%B0%8F%E5%A1%8A%0A%0A%20%20%20%20%E4%B8%8B%E9%9D%A2%E7%9A%84%E8%AA%9E%E6%96%99%E5%8F%AA%E6%9C%89%E4%B8%89%E5%80%8B%E5%96%AE%E5%AD%97%EF%BC%9A%60low%60%20%C3%975%E3%80%81%60newest%60%20%C3%976%E3%80%81%60widest%60%20%C3%973%E3%80%82%0A%20%20%20%20%E6%8B%89%E5%8B%95%E3%80%8C%E5%90%88%E4%BD%B5%E6%AC%A1%E6%95%B8%E3%80%8D%EF%BC%8C%E7%9C%8B%20tokenizer%20%E6%80%8E%E9%BA%BC%E4%B8%80%E6%AD%A5%E6%AD%A5%E5%AD%B8%E6%9C%83%E6%8A%8A%E5%B8%B8%E8%A6%8B%E5%96%AE%E5%AD%97%E9%BB%8F%E6%88%90%E4%B8%80%E5%A1%8A%EF%BC%9B%0A%20%20%20%20%E5%86%8D%E6%94%B9%E3%80%8C%E8%A6%81%E5%88%87%E7%9A%84%E8%A9%9E%E3%80%8D%E8%A9%A6%E4%B8%80%E5%80%8B**%E8%AA%9E%E6%96%99%E8%A3%A1%E6%B2%92%E5%87%BA%E7%8F%BE%E9%81%8E%E7%9A%84%E5%AD%97**%EF%BC%88%E4%BE%8B%E5%A6%82%20%60lowest%60%EF%BC%89%E3%80%82%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(mo)%3A%0A%20%20%20%20n_merges%20%3D%20mo.ui.slider(start%3D0%2C%20stop%3D10%2C%20step%3D1%2C%20value%3D0%2C%20label%3D%22BPE%20%E5%90%88%E4%BD%B5%E6%AC%A1%E6%95%B8%22%2C%20show_value%3DTrue)%0A%20%20%20%20probe_word%20%3D%20mo.ui.text(value%3D%22lowest%22%2C%20label%3D%22%E8%A6%81%E5%88%87%E7%9A%84%E8%A9%9E%EF%BC%88%E8%A9%A6%E8%A9%A6%E8%AA%9E%E6%96%99%E8%A3%A1%E6%B2%92%E6%9C%89%E7%9A%84%EF%BC%89%22)%0A%20%20%20%20mo.vstack(%5Bn_merges%2C%20probe_word%5D)%0A%20%20%20%20return%20n_merges%2C%20probe_word%0A%0A%0A%40app.cell%0Adef%20_(pairwise)%3A%0A%20%20%20%20%23%20%E2%94%80%E2%94%80%20%E8%BF%B7%E4%BD%A0%20BPE%EF%BC%9A%E8%88%87%E7%B6%93%E5%85%B8%E6%BC%94%E7%AE%97%E6%B3%95%E5%90%8C%E6%AC%BE%EF%BC%88word%20%E2%86%92%20%E5%AD%97%E5%85%83%E5%BA%8F%E5%88%97%20%2B%20%E8%A9%9E%E5%B0%BE%E8%A8%98%E8%99%9F%EF%BC%8C%E5%8F%8D%E8%A6%86%E5%90%88%E4%BD%B5%E6%9C%80%E9%AB%98%E9%A0%BB%E7%9B%B8%E9%84%B0%E5%B0%8D%EF%BC%89%E2%94%80%E2%94%80%0A%20%20%20%20BPE_CORPUS%20%3D%20(%22low%20%22%20*%205%20%2B%20%22newest%20%22%20*%206%20%2B%20%22widest%20%22%20*%203).split()%0A%0A%20%20%20%20def%20bpe_train(words%2C%20n)%3A%0A%20%20%20%20%20%20%20%20vocab%20%3D%20%7B%7D%0A%20%20%20%20%20%20%20%20for%20w%20in%20words%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20key%20%3D%20tuple(w)%20%2B%20(%22%3C%2Fw%3E%22%2C)%0A%20%20%20%20%20%20%20%20%20%20%20%20vocab%5Bkey%5D%20%3D%20vocab.get(key%2C%200)%20%2B%201%0A%20%20%20%20%20%20%20%20merges%20%3D%20%5B%5D%0A%20%20%20%20%20%20%20%20for%20_%20in%20range(n)%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20pairs%20%3D%20%7B%7D%0A%20%20%20%20%20%20%20%20%20%20%20%20for%20sym%2C%20freq%20in%20vocab.items()%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20for%20a%2C%20b%20in%20pairwise(sym)%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20pairs%5B(a%2C%20b)%5D%20%3D%20pairs.get((a%2C%20b)%2C%200)%20%2B%20freq%0A%20%20%20%20%20%20%20%20%20%20%20%20if%20not%20pairs%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20break%0A%20%20%20%20%20%20%20%20%20%20%20%20best%20%3D%20max(pairs%2C%20key%3Dlambda%20p%3A%20(pairs%5Bp%5D%2C%20p))%0A%20%20%20%20%20%20%20%20%20%20%20%20merges.append((best%2C%20pairs%5Bbest%5D))%0A%20%20%20%20%20%20%20%20%20%20%20%20new_vocab%20%3D%20%7B%7D%0A%20%20%20%20%20%20%20%20%20%20%20%20for%20sym%2C%20freq%20in%20vocab.items()%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20out%2C%20i%20%3D%20%5B%5D%2C%200%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20while%20i%20%3C%20len(sym)%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20if%20i%20%3C%20len(sym)%20-%201%20and%20(sym%5Bi%5D%2C%20sym%5Bi%20%2B%201%5D)%20%3D%3D%20best%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20out.append(sym%5Bi%5D%20%2B%20sym%5Bi%20%2B%201%5D)%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20i%20%2B%3D%202%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20else%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20out.append(sym%5Bi%5D)%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20i%20%2B%3D%201%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20new_vocab%5Btuple(out)%5D%20%3D%20freq%0A%20%20%20%20%20%20%20%20%20%20%20%20vocab%20%3D%20new_vocab%0A%20%20%20%20%20%20%20%20return%20merges%2C%20vocab%0A%0A%20%20%20%20def%20bpe_segment(word%2C%20merges)%3A%0A%20%20%20%20%20%20%20%20sym%20%3D%20list(word)%20%2B%20%5B%22%3C%2Fw%3E%22%5D%0A%20%20%20%20%20%20%20%20for%20(a%2C%20b)%2C%20_f%20in%20merges%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20out%2C%20i%20%3D%20%5B%5D%2C%200%0A%20%20%20%20%20%20%20%20%20%20%20%20while%20i%20%3C%20len(sym)%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20if%20i%20%3C%20len(sym)%20-%201%20and%20sym%5Bi%5D%20%3D%3D%20a%20and%20sym%5Bi%20%2B%201%5D%20%3D%3D%20b%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20out.append(a%20%2B%20b)%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20i%20%2B%3D%202%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20else%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20out.append(sym%5Bi%5D)%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20i%20%2B%3D%201%0A%20%20%20%20%20%20%20%20%20%20%20%20sym%20%3D%20out%0A%20%20%20%20%20%20%20%20return%20sym%0A%0A%20%20%20%20return%20BPE_CORPUS%2C%20bpe_segment%2C%20bpe_train%0A%0A%0A%40app.cell%0Adef%20_(BPE_CORPUS%2C%20bpe_segment%2C%20bpe_train%2C%20html_mod%2C%20mo%2C%20n_merges%2C%20probe_word)%3A%0A%20%20%20%20_merges%2C%20_vocab%20%3D%20bpe_train(BPE_CORPUS%2C%20n_merges.value)%0A%20%20%20%20_seg%20%3D%20bpe_segment(probe_word.value.strip()%20or%20%22lowest%22%2C%20_merges)%0A%20%20%20%20_merge_txt%20%3D%20(%22%E3%80%81%22.join(f%22%60%7Ba%7D%60%2B%60%7Bb%7D%60%22%20for%20(a%2C%20b)%2C%20_f%20in%20_merges)%20or%20%22%EF%BC%88%E9%82%84%E6%B2%92%E5%90%88%E4%BD%B5%EF%BC%8C%E5%85%A8%E9%83%A8%E6%98%AF%E5%96%AE%E4%B8%80%E5%AD%97%E5%85%83%EF%BC%89%22)%0A%20%20%20%20_chips%20%3D%20%22%22.join(%0A%20%20%20%20%20%20%20%20f'%3Cspan%20style%3D%22display%3Ainline-block%3Bbackground%3A%23E8F0F7%3Bborder%3A1.5px%20solid%20%234C72B0%3B'%0A%20%20%20%20%20%20%20%20f'border-radius%3A8px%3Bpadding%3A2px%2010px%3Bmargin%3A2px%3Bfont-family%3Amonospace%3Bfont-weight%3A700%22%3E'%0A%20%20%20%20%20%20%20%20f%22%7Bhtml_mod.escape(s)%7D%3C%2Fspan%3E%22%0A%20%20%20%20%20%20%20%20for%20s%20in%20_seg%0A%20%20%20%20)%0A%20%20%20%20mo.vstack(%5B%0A%20%20%20%20%20%20%20%20mo.md(f%22**%E7%9B%AE%E5%89%8D%E5%AD%B8%E5%88%B0%E7%9A%84%E5%90%88%E4%BD%B5%E8%A6%8F%E5%89%87**%EF%BC%88%E4%BE%9D%E5%BA%8F%EF%BC%89%EF%BC%9A%7B_merge_txt%7D%22)%2C%0A%20%20%20%20%20%20%20%20mo.md(f%22**%E3%80%8C%7Bprobe_word.value.strip()%20or%20'lowest'%7D%E3%80%8D%E8%A2%AB%E5%88%87%E6%88%90%20%7Blen(_seg)%7D%20%E5%80%8B%20token%EF%BC%9A**%22)%2C%0A%20%20%20%20%20%20%20%20mo.Html(f%22%3Cdiv%3E%7B_chips%7D%3C%2Fdiv%3E%22)%2C%0A%20%20%20%20%5D)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(BPE_CORPUS%2C%20bpe_segment%2C%20bpe_train%2C%20np%2C%20plt)%3A%0A%20%20%20%20%23%20%E5%90%88%E4%BD%B5%E6%AC%A1%E6%95%B8%200..10%20%E5%90%84%E8%A8%93%E7%B7%B4%E4%B8%80%E6%AC%A1%EF%BC%8C%E6%95%B8%E6%95%B4%E4%BB%BD%E8%AA%9E%E6%96%99%E8%A2%AB%E5%88%87%E6%88%90%E5%B9%BE%E5%A1%8A%E2%80%94%E2%80%94%E5%B8%B8%E8%A6%8B%E5%AD%97%E9%BB%8F%E8%B5%B7%E4%BE%86%EF%BC%8C%E7%B8%BD%20token%20%E6%95%B8%E5%B0%B1%E4%B8%8B%E9%99%8D%0A%20%20%20%20_xs%20%3D%20np.arange(0%2C%2011)%0A%20%20%20%20_totals%20%3D%20%5B%5D%0A%20%20%20%20for%20_n%20in%20_xs%3A%0A%20%20%20%20%20%20%20%20_m%2C%20_%20%3D%20bpe_train(BPE_CORPUS%2C%20int(_n))%0A%20%20%20%20%20%20%20%20_totals.append(sum(len(bpe_segment(w%2C%20_m))%20for%20w%20in%20BPE_CORPUS))%0A%20%20%20%20_fig%2C%20_ax%20%3D%20plt.subplots(figsize%3D(7.0%2C%203.6))%0A%20%20%20%20_ax.plot(_xs%2C%20_totals%2C%20%22o-%22%2C%20color%3D%22%234C72B0%22%2C%20linewidth%3D2.4%2C%20markersize%3D6%2C%20zorder%3D3)%0A%20%20%20%20_ax.set_xlabel(%22BPE%20merges%20learned%22)%0A%20%20%20%20_ax.set_ylabel(%22total%20tokens%20for%20the%20corpus%22)%0A%20%20%20%20_ax.set_title(%22more%20merges%20-%3E%20frequent%20words%20become%20single%20tokens%22)%0A%20%20%20%20_ax.grid(alpha%3D0.3)%0A%20%20%20%20_fig.tight_layout()%0A%20%20%20%20_fig%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%E7%9C%9F%E5%AF%A6%E4%B8%96%E7%95%8C%E7%9A%84%20tokenizer%EF%BC%88%E4%BE%8B%E5%A6%82%20GPT-4o%20%E7%B3%BB%E5%88%97%E7%94%A8%E7%9A%84%20%60o200k_base%60%EF%BC%89%E5%B0%B1%E6%98%AF%E9%80%99%E6%A8%A3%E8%A8%93%E7%B7%B4%E5%87%BA%E4%BE%86%E7%9A%84%EF%BC%8C%0A%20%20%20%20%E5%8F%AA%E6%98%AF%E8%AA%9E%E6%96%99%E6%98%AF%E6%95%B4%E5%80%8B%E7%B6%B2%E8%B7%AF%E3%80%81%E5%90%88%E4%BD%B5%E4%BA%86%E7%B4%84%2020%20%E8%90%AC%E6%AC%A1%E3%80%82%E5%AE%83%E5%B0%8D%E4%B8%8D%E5%90%8C%E8%AA%9E%E8%A8%80%E7%9A%84%E3%80%8C%E5%88%80%E5%B7%A5%E3%80%8D%E5%B7%AE%E5%BE%88%E5%A4%9A%E2%80%94%E2%80%94%0A%20%20%20%20%E9%80%99%E6%98%AF%E5%B7%A6%E9%A0%81%E9%96%8B%E5%A0%B4%E9%82%A3%E5%80%8B%E5%AF%A6%E9%A9%97%E7%9A%84%E5%8E%9F%E7%90%86%EF%BC%9A**%E5%90%8C%E6%A8%A3%E4%B8%80%E5%8F%A5%E8%A9%B1%EF%BC%8C%E8%8B%B1%E6%96%87%E5%B9%B3%E5%9D%87%204.4%20%E5%80%8B%E5%AD%97%E5%85%83%E4%B8%80%E5%88%80%EF%BC%8C%0A%20%20%20%20%E4%B8%AD%E6%96%87%E5%B9%B3%E5%9D%87%E4%B8%80%E5%80%8B%E5%AD%97%E5%B0%B1%E8%A6%81%E6%8C%A8%201.1%20%E5%88%80**%EF%BC%88%E5%AF%A6%E6%B8%AC%20%60o200k_base%60%EF%BC%8C2026-08%EF%BC%89%E3%80%82%0A%20%20%20%20Token%20%E6%95%B8%E7%9B%B4%E6%8E%A5%E6%B1%BA%E5%AE%9A%20API%20%E5%B8%B3%E5%96%AE%E8%88%87%E4%B8%8A%E4%B8%8B%E6%96%87%E7%94%A8%E9%87%8F%EF%BC%8C%E6%89%80%E4%BB%A5%E9%80%99%E4%BB%B6%E4%BA%8B%E8%B7%9F%E9%8C%A2%E6%9C%89%E9%97%9C%E3%80%82%0A%0A%20%20%20%20%23%23%202%EF%B8%8F%E2%83%A3%20Embedding%EF%BC%9A%E8%AA%9E%E6%84%8F%E8%AE%8A%E6%88%90%E5%BA%A7%E6%A8%99%0A%0A%20%20%20%20Tokenizer%20%E7%B5%A6%E6%AF%8F%E5%A1%8A%E6%96%87%E5%AD%97%E4%B8%80%E5%80%8B%E7%B7%A8%E8%99%9F%EF%BC%8C%E4%BD%86%E7%B7%A8%E8%99%9F%203421%20%E5%92%8C%203422%20%E6%AF%AB%E7%84%A1%E9%97%9C%E4%BF%82%E3%80%82%0A%20%20%20%20**Embedding%20%E6%A8%A1%E5%9E%8B**%E6%8A%8A%E4%B8%80%E6%AE%B5%E6%96%87%E5%AD%97%E8%AE%8A%E6%88%90%E4%B8%80%E5%80%8B**%E9%AB%98%E7%B6%AD%E5%90%91%E9%87%8F**%EF%BC%88%E4%B8%80%E4%B8%B2%E6%B5%AE%E9%BB%9E%E6%95%B8%EF%BC%89%EF%BC%8C%0A%20%20%20%20%E8%AE%93%E3%80%8C%E6%84%8F%E6%80%9D%E7%9B%B8%E8%BF%91%E3%80%8D%E7%9A%84%E6%96%87%E5%AD%97%E5%9C%A8%E5%90%91%E9%87%8F%E7%A9%BA%E9%96%93%E8%A3%A1**%E8%B7%9D%E9%9B%A2%E7%9B%B8%E8%BF%91**%E2%80%94%E2%80%94%E9%80%99%E6%98%AF%E8%AA%9E%E6%84%8F%E6%90%9C%E5%B0%8B%E8%88%87%20RAG%20%E7%9A%84%E5%9C%B0%E5%9F%BA%E3%80%82%0A%0A%20%20%20%20%E4%B8%8B%E9%9D%A2%2016%20%E5%80%8B%E7%9F%AD%E5%8F%A5%E7%9A%84%E5%90%91%E9%87%8F%E6%98%AF**%E7%9C%9F%E7%9A%84**%EF%BC%9A%E7%94%A8%20embedding%20%E6%A8%A1%E5%9E%8B%EF%BC%88jina-embed%EF%BC%8C1024%20%E7%B6%AD%EF%BC%89%E4%BA%8B%E5%85%88%E7%AE%97%E5%A5%BD%E3%80%81%0A%20%20%20%20%E5%A3%93%E7%B8%AE%E6%89%93%E5%8C%85%E9%80%B2%E9%80%99%E4%B8%80%E8%AA%B2%EF%BC%88int8%20%E9%87%8F%E5%8C%96%EF%BC%8C%E8%AA%A4%E5%B7%AE%20%3C%200.006%EF%BC%89%E3%80%82%E9%80%99%E6%A0%BC%E6%8A%8A%201024%20%E7%B6%AD%E7%94%A8%20PCA%20%E5%A3%93%E5%88%B0%202%20%E7%B6%AD%E7%95%AB%E6%88%90%E5%9C%B0%E5%9C%96%E2%80%94%E2%80%94%0A%20%20%20%20%E5%9B%9B%E7%A8%AE%E9%A1%8F%E8%89%B2%E6%98%AF%E5%9B%9B%E5%80%8B%E8%AA%9E%E6%84%8F%E7%BE%A4%E7%B5%84%EF%BC%88%E5%8B%95%E7%89%A9%EF%BC%8F%E4%BA%A4%E9%80%9A%E5%B7%A5%E5%85%B7%EF%BC%8F%E9%A3%9F%E7%89%A9%EF%BC%8F%E6%83%85%E7%B7%92%EF%BC%89%EF%BC%9A%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(np)%3A%0A%20%20%20%20import%20base64%0A%0A%20%20%20%20EMB_WORDS%20%3D%20%5B'%E4%B8%80%E9%9A%BB%E5%8F%AF%E6%84%9B%E7%9A%84%E8%B2%93'%2C%20'%E4%B8%80%E9%9A%BB%E5%BF%A0%E5%BF%83%E7%9A%84%E7%8B%97'%2C%20'%E4%B8%80%E9%A0%AD%E5%87%B6%E7%8C%9B%E7%9A%84%E8%80%81%E8%99%8E'%2C%20'%E4%B8%80%E9%9A%BB%E8%B7%B3%E4%BE%86%E8%B7%B3%E5%8E%BB%E7%9A%84%E5%85%94%E5%AD%90'%2C%20'%E4%B8%80%E8%BC%9B%E7%B4%85%E8%89%B2%E7%9A%84%E6%B1%BD%E8%BB%8A'%2C%20'%E4%B8%80%E5%88%97%E9%AB%98%E9%80%9F%E7%9A%84%E7%81%AB%E8%BB%8A'%2C%20'%E4%B8%80%E6%9E%B6%E8%B5%B7%E9%A3%9B%E7%9A%84%E9%A3%9B%E6%A9%9F'%2C%20'%E4%B8%80%E5%8F%B0%E5%85%B1%E4%BA%AB%E8%85%B3%E8%B8%8F%E8%BB%8A'%2C%20'%E4%B8%80%E6%9D%AF%E7%86%B1%E6%8B%BF%E9%90%B5%E5%92%96%E5%95%A1'%2C%20'%E4%B8%80%E5%A1%8A%E8%8D%89%E8%8E%93%E8%9B%8B%E7%B3%95'%2C%20'%E4%B8%80%E7%A2%97%E7%86%B1%E9%A8%B0%E9%A8%B0%E7%9A%84%E6%8B%89%E9%BA%B5'%2C%20'%E4%B8%80%E9%A1%86%E6%96%B0%E9%AE%AE%E7%9A%84%E8%98%8B%E6%9E%9C'%2C%20'%E4%BB%8A%E5%A4%A9%E5%BF%83%E6%83%85%E5%BE%88%E5%BF%AB%E6%A8%82'%2C%20'%E4%BB%8A%E5%A4%A9%E8%A6%BA%E5%BE%97%E5%A5%BD%E6%82%B2%E5%82%B7'%2C%20'%E6%B0%A3%E5%BE%97%E4%B8%8D%E6%83%B3%E8%AA%AA%E8%A9%B1'%2C%20'%E5%9A%87%E5%BE%97%E8%BA%B2%E5%9C%A8%E6%A3%89%E8%A2%AB%E8%A3%A1'%5D%0A%20%20%20%20EMB_LABELS_EN%20%3D%20%5B'cat'%2C%20'dog'%2C%20'tiger'%2C%20'rabbit'%2C%20'car'%2C%20'train'%2C%20'plane'%2C%20'bike'%2C%20'coffee'%2C%20'cake'%2C%20'ramen'%2C%20'apple'%2C%20'happy'%2C%20'sad'%2C%20'angry'%2C%20'scared'%5D%0A%20%20%20%20EMB_GROUPS%20%3D%20%5B0%2C%200%2C%200%2C%200%2C%201%2C%201%2C%201%2C%201%2C%202%2C%202%2C%202%2C%202%2C%203%2C%203%2C%203%2C%203%5D%0A%20%20%20%20EMB_B64%20%3D%20%22b1LJEEUF4%2FAsDaECANqy3%2Fw6MhAC9Pas3%2B44K39OAAfpCPflEysZ2%2B7TPM0KTh63HB7q4N1BDv7n8rvxzwnpTwTq3PM5%2BxHkzecLxxRKFCLk3yTzAeYMBt8QxPUvCQ7tIw8X3fUS6vL8CyAMt%2F%2Fv%2FhDps94b5PE8wcvQDsfu9PYQHePHDwju9yT44x7gA9HwEz7bzN0Vbg7lCRIXBQgcLxwx%2F7ng1fT96cwBCsv3Q8LoAvgNEvUJzvkm1AgK5uskzQIUAig3%2Bw0FK%2Fj41QIAAgQgBhMB3emG2OP%2F%2FLvh7gMz4BgAB%2FPpBATc5wkt2BrhGgnj9BPo6vE0wSokA%2Bod8PzeB%2FHsJyX0%2FgQB%2FPvuIvIl9QwyGuv5EPjp5jb%2B4iX6URHpFu3p70Ya9csR8yIu8vQAFQ%2F8AgwK%2BP7d4fM01%2FYA9jjwFOrx7AL1Ee8sGtbo4%2FPbA%2BD%2F5t8hxR7Z2%2FxEAyHGvLLCBuER5vrd7N3O5g7NQfwB7i48B8juBkgJ%2BP%2FeKRoI%2BAr8%2FAca%2BdcAOtA3Ggi0GtQdDgQRAv8PIf4MEfrd7wobF%2FbpDM3YyAT9%2FugL4er0DQcOBvb91tnJ7hYE%2BgkECjXzBhICHgAGAfEONQnvM%2FbkMOPTBhG0BRY%2F6fLw9e8VBQHz8%2F%2FnJMsR7uHGBfDbEPXpLBcS9hUIEPbyxgbXFfPu8x0G5RmqBdMN9vTh1g4i4xT%2B9hHs%2FdjsFjkOA%2Ff%2BCA4MMhHP4%2FUR7QMtHwf%2FNODdAfXu2wcOzfYUBNPc7%2BskL%2Bvb5%2BAYHPTuC%2FgNDT%2FuDusSExAHMOX2I7hA8A8c8BAN%2FAozLPn0Bx8kvxrpAO0y%2BxAY7FQV8gsG%2FNIGFxM%2F%2BwzGHCXc9%2FHm7RoQDeDpBAEeGgguwAfpCfncJtJR%2B7%2Ft6ubfAv1G3R8KHf0H6eb5ECPi2%2B7lNBEpKjoG5gMn4%2Bn0%2BtPz%2FM4BShYV7g3yKgU%2BBRj35hsH8hHzExMH7e3b8T4E2831BzT84wgb5PmyEhdAvvUDGvft%2BhQIEgwS5PTj9MkbANbKH%2FkG8vTl3%2BoHue4Y7C82y%2FLZLdsY7%2Bnq%2BPX2BgMxHf%2F%2FKAsr9Mrc%2FAME5RoxH%2F30ztDU5x7y6P3E9AH%2FEevw%2BRvtzATw9Af%2F%2FBD7Ff%2F8FwD35frtGQ0zL%2F8D%2FQrb3%2BPWF98iMywLEewJ5t0l2iID8EA7HhA83B3W2Av4xTn69%2BX9GuvnJ%2B%2FqCBT%2BAR3T8RUE%2Bd3uJfkMEtvdDgvGSRD40uLh%2BTsSK9oZ8ubm0CUaDAoVDiH1KtriFCz3Ix4L3xLj6N0JCuzOD%2BHl%2BP%2Fz%2B%2B8N5RLm5u3z4RL849jIEOYFFecoDjjkE98bxvEvACpH6eICpzrTIxowG0Vr0SVbztP%2B3BzxxMLbyzYJMz8JCgbBgRHm%2FChIZwcECybx4iNVRp%2FV5SDmI3UJ4ib2FO9CMRYT%2FubuKu0QCxgSHS%2FCAfwkHSXuBd0vVRk3KeFABRHcGwHdIszcJEL%2B%2FA3dAu319z0C%2FgQh%2BtnJHwQbutAANQ4XMcbC6PHFDPX6GhIE%2BgQl4u8B2Oni2wcGBuj6O%2FTeAVUTye3AFiIs8wDzDiT3FubT%2FBwB2QTlHxnz6uXGCeASHgHQBucpA%2FwAC7jrMTG7MuEu81QKNuEN9PwbCyv6KMwWxuzp7THW%2BwUJSB8yBRAFzgwKCN0N4fgUIPvx%2Bycy6hfL48HUCfMi7NoSsyL39wcR6tkG%2BB8s1gX6CeIbJy73Nf4UJvkeFeZFCQ0c3w7kteL5Q9MCJgsXKuP2Gf0C3%2BD0FPv7P0SzGwI0CAL46j7qIxK87SH9Lvvn0%2BsC6A7wGt7sD%2BUWHtMLGb4A0KvX2Qr5MPUc2Tru%2F%2Fg67%2F7%2FEOoE2v9J4f4f6wzr%2BhZDBwwCQggx%2BNYAuNzxDejH1iv3CgHqLf4aGPQMBy7w%2FeY3FQnb7P8a0%2BQV4db00gb3AuUEDgcUAuj3NfIDFukO6CAcOtrp5fs9GugVLj4E9BwI6wvgzNn09y7zFPv17A7xARLu3Pca8hXjFMUTutHG%2Bv%2F58xAaFxFCEhMAydTr%2FfIJ2NL5IuMrEgq5EQk0Hef75PXw%2BvDS3NHBNij1MQkMvAK9DFzm%2Bcr0G8b9yRHkLBzkCwP7ERooF70Uztwq%2FyTxHPMfAAnU4jfoNurwCQQfKSXS%2BQUoGg3UFBrLMO4F7vQkBwU85%2Bbz%2BwkE6tocLAP4Igz6IvkFBgv05tvO9CYQIvv%2B0Ac0HBrhwPAtKB0J7vXuCC7oV7f%2F5BPxLEvbIfP%2F%2FNAi%2FdEILu07GDv1%2FQMa8gEEPPcF6QD6JRccE98UM%2B%2F30f7z7APdI1AqJxkjGjwC9OgF5fcSD%2Bf8DyAN9fkx1wA5PjEB1xf9AUzTGuPt1R0T%2B8EjSgrJBwAYGe3XC%2FoE2ei%2FPvkM%2F%2B3bKQ00ENkeG7jzEAtAdd7jyB%2FrCwQ45BQAEQD7NP33%2FBXHItP%2F1OBA%2Fxgq%2BgzZ8%2BXPIeQC5AMEzxAK0Ujs7tXXJ%2Bfe7PIu9ggR%2B%2FwGNwP8GBHp8dkFKiDtIfLE7uLX5O8LYRvIFNT27izs%2B9sDGScxGPINTgHy184%2BFNLy%2F%2FX75%2Fz55PDiGe%2FWGipE8w%2F3xPzN8jjoL%2F0KBQ%2FbxTHoGcTgKffs5vUZEd3GAusvPO7W%2Fw4NHxvL%2FSomJzA4B%2FAU%2F9Uv4fQZ9Qns3ttPyA4HG9b9%2BOz82%2Fvx%2Fu7U4v3dsg3v%2FAUU8v3EMr0DBRVJNOLx5%2Bc2JyUqHeFJf9MAO%2BP9%2B94QyvLj19ILFAIlJQYe38cDFyY2OV0rDBE27eYJDxPm5y4X09I%2FOAAn7BAdHBMHPAnP%2FvsLSsAUGRoJGyoBGOgUAQHiPhAa%2FQAx%2BPUF8%2FHwz%2BvL%2Fgy76OQUFPn%2BAuzg%2BiEa5ATp8hUJ3e70BxcB%2BTDV%2BiL4%2BQTi9xcQGBnvGAsJ5xL3CCEMDgv%2FD%2F%2Fc4%2BUaD%2FwNIBA3%2BAEGDusV1%2FrkBgQeBQT68g71ENHw3%2F4P7hnd8CIhBf%2FqCfXJ9i0s7xH1HeIm9%2FP2EQv95%2F4nBxXQJObC3foOI%2FThDioHERcoOPMV6CAV2fDwEe0cKQT6%2BPH59ePhHwXz9fH3CPr%2FD%2Bz4BwDvAxEA7Pr88iD6BwgH%2Fes9GfDxDAzZHuUlDvMVAdnX8Q7P7ywBDeP%2FEfUI%2Bvrz0gQL%2FwEEIyIK%2FQXg9OsoKRzz%2FQBCAfAYAQXvDcoJ7A8Z1hfl%2BOr%2FFykpBCvH%2BuwX8BPxAwk%2FAdj9FiEXBfEa5snPAej7DA7w3SToHOwQBgv77Q4YAgv7sgH32QUdGRUC8Sci6CgXGPTc6%2Bbu8REXBvfwz9%2FwQerZ2wHeFtwtER8C7UHkCQvs9gIG5P31G%2B4DDfQSG%2FwMHxf3Jxc5Eg%2Fr6foB9fjvAAf8FyvZAyQRCfEN7tsVFhzw9Sv579vqAxsv7ur0DRQj%2BBjp6QooFRb%2BNA3qCM8p8PLUERXp7Q0M8RoS7vj6zOTLGhfd7vAw%2BQUc6wq%2B%2FfPRx%2FstDfgP4gQL8dgPB%2FsS603ZyvgU7gvaJxLp2fQB9eEzBhYHMhTzxvEiJxgB%2BOwHxOwxDAgRDRQs7iP96yBBBwDnIOzh0Tj%2B7gnk7N%2Ft4%2Fr6CwcqDBIH7iH28BD42SHo5fvO9gAOCwAHIzLp%2FdvfH%2F7nABPrEisZEf3w2T3i2hrjBPnoGxzdBv8yA%2F482SPzLBIIKhD5FOYl5Qra3OYH4%2FoX%2BDQRy%2B8GEv3rCwbkFfTl8TEHLP%2FQ6fwWBegB9enuyRLm9S4t3%2FAc%2BDj96fbHFu4s%2FgUH%2B%2Bv5BCzi%2BAvmAeT0MwIR8AX7JwjeFhYa%2BQnK3NDj7eQd5jIDCxYf%2Fv74H%2BH%2F%2BOGtFPYf2xQjB%2Fb5Dt7A2Ob5%2FiEw%2BfAM4to0Cxz50eYnAhUCCgYAGgfW%2BxXkBr%2F5FikLCh8b4vzP09TTChwM8P8jDR4ADPL0%2ByEQGfPq%2FCAAIfbhKSMf8ffpCdv87fj88yjy%2FBMXDwHkD8jsCAQhOhMH5hEL%2FcUdCgPH7AII6gvvCwEe3g4HJBTv6y%2Fo9CoQx%2FYBGfz1L%2Fr97P%2FjMfz3CwPkurrg4s7x%2Fx7KLOkO5%2FIN9cb71%2FgQH%2BE34dXuCSrw7SgN%2B%2B%2F9Dh%2F49AnxDPsoGSAOPCrTHSzs4%2FzDFckX4Nik7BsNOC9GF9%2FRGRYuE1V%2FB%2F0QJOnk7hIO4vX2DREEeAQN%2BvgC8BYlECjc7Ork%2BxYBOOrk9dPoFiAn1P782fA7BiMbAhcIDxM0DhZNrdbzBObaJQgKAO36AgoWE%2Brh9BQM1vXf8RcJ8P8H4sfa8%2B3a2%2BsO%2B%2FnyDsL4BBgRARn22hff9xYP09EAExLJFgwF9RERC%2F8JCLb%2F5SgUANzn2tQKFe4E%2F%2FAo%2FPf1MOgmAyQX9Qcg%2BvUXCRM1AuTyIggM%2FAnyCxz%2F8QD%2FGwnT%2FxMRB%2Bb7BvwvEPYYAO3xCAn7D%2Ffn4S%2FHIB3m7h8Q6g8H%2BukSy8UH%2BOzv9PDP9Rf65BoCBgcPGegj5RUsHt63FA7dDBwA8yjjIwDRARgHG%2F%2F27O4QIAr28tQW9evn9eXo2RDUBN%2F%2F9uXF%2BgUV%2BBb5D%2FzbBvwN%2FPTcIQPjNt4w0w0l9Qj6ERYnHBD5rwXaAu0%2FHN3THvznCP%2F2DDf51hkcBw3N%2FRImFPHq7lMAIx4IHB8N0ggRINIaD%2FOuIeURCRrZGQ%2F8Dv%2FsHO4MHw0JB%2BbyMdj06S%2Fw8tjpC%2FLyAuLb9OgR8uoiGefs5v0OEB%2F5BST8GwXv5BktGwn7By4H%2B8%2FSDhfYDwzuRQf59%2BomBNPz5PkZG8gQCvv9%2FgTc%2BgDuDebx6BPtGvT389fxAwzv2xfl8A7uBfjvAt%2F1wwf%2B7gUZ%2BysYvf7v1iQE8gcZ9Rf8AS4B1A396%2Fjy5%2FvYJQsM%2BNbwtQ35whP89fnLI%2FcNSuX1IuLn%2FvDjDhUPGOwIF9wtEv8M8%2FQmD9QA8f0cNhgA%2BPbtCwkRDeAX0Q36IuT19OgO7jq0C%2FL89eoU9AcL1%2F7x6Urj%2B%2F%2F61ysEA%2FUN9%2FAOESAt9Bn68vf8%2FdYL%2BgAS%2BQEi9Awr7fgVAwokxRLtCQ%2Fo6%2B0UGyH24Qv53AAHzcrZCsHj%2F9%2FbLBIBNSDpEvsPzCoVBiX3%2BOzqIvX1HQPZ%2FA37D%2F337gUSDNkKuSbzBuMszPgiHeLp2SfuFfzc%2BO%2FOJvQC7cj%2B%2Bxru7%2FcbAA4H%2BQ0RGSwtCw77%2F98dyPAC%2FxD7FO4H5iIL3%2Fz2vNsCDe8G6%2BAFCNfb%2Fe4ep%2BP8zwDsHOn8%2FO%2F%2B%2Bx8T9Pf12QH%2FCP7gBBX1BBfuz9%2F3BPVCKg4mBfb2CBLHLQYdECjtCOQS6v8DBBgoKewa8B4qAwPN1C0Vx%2FwE7%2FkhCuvx3SMdGiIZMhwPJePhD%2F7kDx3jB%2Bm9Nv29DLwpvfP9ABITJO76%2FQfqBQoeAuT68QQFLAkEKv8p7RkJA%2FgiBwkVDRgAFSXBAPME8f4Xxzb07%2FnB9P40193yCcv68PcvFOcFD%2BYI6f0G%2FDEo9fsQ3vcFFx4l6g9%2F5ApfIhzi7P8B%2Burs4QrG2y855uf72h4sH%2FU6CAwEBgXv7vbv%2Ffgr5ssC%2FAQQ5vEIAeEJKAou5t3%2FAOsyB%2FXpBBbv%2Fg8C6vLu6%2BfsKPb%2FEhXW8Bvl7f3n%2B9f8%2F9vx2v8RBP%2Fn%2Fwv06A4J8RgaCAAH5gL3B9DoEOjk%2Fg%2Fg27%2F%2FBAELCO4B7v4iEPsXGw%2FtKe7%2F%2FgEFJQoB%2BQ%2FlGgMUDAUJ%2BvYFzhgIFwUK0%2Ffo8yT69%2FvdGA7iEOb6%2Bg0P%2FPfuE%2F349AoDEhUI%2FzHqCAH0LPcbDy4BCRIN7QAiAxb%2F9vYAKAX1CQD3AA0CCuzjFv4IH%2FISDhII8ujr8%2FMAAdvd%2FwUT9PsA%2Fvgh6BsI9gcPFOYFDxb62uEN8CkaC%2FT7CPME3Ow63yEYJBkK%2FMgBGgIW%2FP01Dg8A7Bbc9dziLQDq%2BwLz7vUg6Pb68ekVDOL7%2BM4cA%2FTM7CvZMR4r%2BAfX6Bz79wv9J9n2%2B%2FPe%2BucAJTnjE%2FfnGvUR8Boe%2BPI27CsFGgEAAhYVH0AP%2FhHv%2BhUHE%2FHG9QjF6Q393gHvEwb%2B6OzwIP3h%2FPTpExkAFyX%2BCMjh2PsI3NPI%2F8AI6xvdEPHfDPL9F%2FP0MBD%2F8e0M4uwFAgEFAQ8y9yEp%2Fv0KGd%2F16vvt1QXUDvTEDwfv%2Bdcp%2F%2BzIIPrm5%2FkJAwEY90rxD9EbEvgDE%2FL3CgMUCtkQEvT47RMFIQvsAvoTJxcUGOz%2B6%2F3p%2FgDk%2FAkb7OLq6PPm%2Fu%2FjDfnb8fsh%2B%2FTg%2FBEBFQ3l9OIaGhHX3AvwCOr37Owf9hQI8QrFAecw5%2B8H%2FRIH%2B%2BHaBdwIMgr59P%2F%2BJvP%2FIfUtIeTi7gYa7%2BMWGeYFCvDmBer9AP375%2Fv8Agf0CCj0AfIP6AML8hTs%2FS7g%2Fw8KFusSBff1CeUdBNsE4hTq3eQHGv4AGCoE8%2BMKAPAE4fjtHyH7BQYH8RcG7hUU8OgG9gX29RIXEN8G6BEQ6u0JB%2Fb8ChfyCBUcF%2F8T4%2F8R5PL91SgEE%2Bf05yYG9%2Brg%2Fv8OEgX8DuL4FAD4Bu7u%2FerpAhn0%2BvIJGN4D5Pb9ANkj%2FSru9%2B7qAeDxFfwOFQ0FNxErARD%2B%2Fecy%2FfDoDyfqHAjpDjP4%2BsUHCRrMA%2FcMCxbc5AnP7xkS8N%2FeDRIFGv3g7uUZCRj3CPn13gn8GCDX%2BQ%2F5Dtz%2B4xblF%2FwaFQEE%2FB7w7vgBLO8IIf71EAfj8BcaDvMa%2BAv8Id8NA%2FP77PDrAw3%2B%2BBEH4BUp4%2FIa%2FfgN7SLl3OMCA%2FX7DvDoHAgUKd8CFxse5gD089337ifrEPH35ysQ7%2F%2F%2FFvgL%2FhUI%2Bw7o%2BPEH4uj1%2B%2FwDEQYK4%2F8I%2B%2B75%2FyYI9e4A7DALAvPm9wTSB%2FkYD%2FbeLf4r9xP0A%2BzzR9EKZ8Hmgcj%2B9fj93sz7DR4lEs3p6BBPRmoxRiLn5cnpyuof6R%2F%2BN9%2FnIQsrABXs8OYdIDgoRe3i59%2F6LefjAiAgDfbrSuPcBf3lywbn8xT9AxwZFtvqJMHyyRW80uIR2AgZEf8A397bw%2BP5VND3Gs0c6jP28SvS5zEXDuHS7vHjUAnz2SY%2BKQHbE%2BAM9CoUEeQV1vf246zKIuztEQAqKdwNPdDmGAj%2BB%2BQY7xYK8gbuFODzAd0a7BEiDfL2CQ%2Fw7e0X3fXP%2BBokARchvQTnBwEEMPr3MhzkJgzuL%2B8R6xctAes0BAAa%2FB3p7APiCvvs8M7wLicY9gEX7vsK2%2FMb4vHfKefKHQkHI%2F%2FxEPTXHugQ%2F0kB5Plh1dlHGey29OEiA%2F0SFvn%2F9PME7%2F0K%2FdQADfca3efi1RH%2F9hHy8frc%2B9rc%2Fw7tPu%2FvBCQoCg4UAx4B%2FLfr9STQ%2BOXeLuAL3sf%2FLOO74yQi9D1T%2Fcb4%2BDAc9PT76hf24OnjHvbsKv0W1gDsD%2FMz9xcb%2BCrIE%2FX4wwDnCPAF%2B%2Bf36%2F7S7PQU7RDy9xfhFPHsDfEE3gTg8yL4BwXDtvDH2ibq9Evh17EXDB%2FnyRkF8RkE5gzSE%2FvwFPETGNQqLfQSEA3V1vzSwvEn%2FSgQECr80gjw%2FAPc2QfeACUPIOsa3QoH%2F1P9DwIO2zLK5vri9xIAFUnm9AQE8egAywcfFBzN1AQkEznS3Q7IFwgV%2Fa0RDrXt9ybu%2BhIg5d7iC%2BoaJgw25SUEC%2BkO7BEVDMMOCfwG%2BvrzC9zHBu8nW%2BoXKt7t6%2FsQ2MEMEeUr%2Bdfo9hnK3%2Ffb8EkJDAYyG%2FQTGOXw5fyiAtQbDA8mJA7ZAefw8wuoFxQRHhvnxyG3Cyrl%2BxIC3BrGAybk%2BTn%2F5vTLExsv%2Fw4m5hvjNBkM1QoSCMTsDAz%2FBfMJ%2FOwsG8sK%2BfgiHD4k9QQw8f8D3P7f3NTyDQ7bERXtAgMOyeTw2Ano9OD%2FCeXT0u%2FlDuz42fv6ERVFCAD%2BAcrP%2BxcHBAnV9%2FPqyfwyC%2FHfAyEo%2FPr7Gy8V2eUbHg%2FUNcIL4h8fB%2FnE8fP5IuX6QCTt%2BhAPEhjzVQgF8ADx88wRAfQ29gktCAgP6Q74DM%2FTJw4I3hXYAcb45i9T8wPW6vQqAUzuJt75%2F%2BsDEyT3AgjUK%2Bnd%2BRQDGggDFgsc9O4Z%2F%2FLqDuoQDPDw3vkmIAYCNPP6AfdINtsu9c4V8wMHMQP89%2FP1CuD68dL58eoo9QFH5g%2FuGdwB2PIa9SspHhvm%2BPPn%2Buvv1QkQvALfCTIHG88WMOS89d7K7Ovh277y2yAJ6RD13%2FgQHwbx79kABv4W7uT8NNgH8dgKB%2BzmCAHr9QfrGtW%2BEQkaDCEOxKrQ8DHPwH8I%2BIKX5djc89iNFtXPTSsAPwvqMWAlKGJCNQzl5A7g4OgK9zzXKBPwUQn6HNf4%2Fe0jFlTd7Qn%2FA3QBEfgL9AoU%2FgcYtwEJxOTg3gYIDQfWDBEQ9OoD2Nbr%2FBfk6vX%2F8ugF974d2Aff8QIICNzEbf4B%2F84R%2F%2B%2F96av%2F5%2FTcH0cECMDsAxv%2F5ALxBPwc9dHZ5fDr5AO9EPf2%2BS33K%2F3w77%2FX%2BisA7xX5At8TAQH53PnpHvoHJPnQ7O8NHho8%2FgI06AwR1erjwRb5Ae%2Fn7Brj%2FusbDREQ%2FPr7%2BgP9Ow8pMuMt7vfsQ%2FUM4RcKEyIFCgAYGREh1%2BgH8DMGOg0SNPcw8hf5KOYL%2BhQI7gQY4fgmIBUvCQIWKC%2FyD%2Fbv6xL%2F9hTc%2F%2B8XvwHx8vsvLvm39B4NBfwE7gXDyhQy%2Ff9XA%2FTxF%2BgS%2B8MVGeTrFBb7EQAYEf79GAwg5Efx0Rnr3xb6BSkAuTvcJgj76uLEMw4CE%2BH4E%2Fz12vrcAOLx9ysNBx4S9xEF5gkVJQHm9R3m6AbqY7%2F2Iu7rAg8y3CTUHAfmAO8QGxb99iAOV7nvvxUS6xPR6rr1%2FQPO4zAiHfQV6g8aG%2BBSGhIV6yfZH%2FPUIvMf8i%2F7HgLSHN%2FU%2BLfhB%2FL5FQ4c%2BR3IBwMPzRkH%2B%2FvuBOkAAPzs1w7sQMjw%2FRLmEen%2B2t7s8uYW9VH4JOsd%2FSwQAefv8wz2CwgO9Ob%2FyOgC8foDCPv8Af3sNPzTEvQ%2B2gH%2F%2FhrN7%2FAS58v5AebuCO%2F88ikD3yHz%2BuPvLf4BBijyFusfFhMOCu7yBPIJ%2B%2BFFBN3SGgIp98AW48cp7%2B0HG%2FId1e0YHfnmEPreDxH%2FD%2Ffx8%2FIOThb1A%2FEG5e7N2hL9B%2Bvq%2Fx8PHOzr7Rz37Ts01PUeDK726i0MsAL38QgA8%2F%2FoChP8%2Bykw6Pq8EPbvHSkX3%2FfxBdUg3AcFF%2FjIMPzpE%2F%2FnB%2BDPJxEdGvD3%2BxTdAeva6%2F71AwHz7%2FrV9ivj5vpFHPnxOdwZCg4h%2FfzyB9nvwuct797rBOAvDgT1%2FM3szgbnF%2FT3xfUL%2FhQEEA4j5w8Sz%2F3%2BAxkSB%2BwDLEgB6Pf31gEBv8EkBf36D932%2BwgJLgEF6%2FLSC8MM%2Beci7ucNI0D8GdsQ9yPc%2FPAHH%2FbyCwQ6INoc5hoJ%2BdwBE%2BMHCC67JNgR9%2BIEIg0E9%2BL8AOQbEhgZ6fUwBCoFIAPYFusK8wbn7gG9EOwQ%2BjQMP7AWEBDe1ukSBvHeEOvhEvPp7M8L6%2Fv39NcI8R7e9Ef2%2B%2FnhIO0O3f8Mz8oA9voZ1fHrABHxJRXbEwsJ%2B%2FQcB8grKQXi2%2FHc8SutCPD2xvDo6QAV9vwE7wgSBgDn5vDy3hv7XvcEEw0TDP8a9dI2vbpBGBUOHybT3fHF50AfFEw5G7ThEuEZZBV%2FNT0K7Ov%2B1znoIfH1ywsABvz42%2BTzBPT%2FMPkF%2BAMCEPQXCg8FDAH39Q4vGd8BDfwf7vEvxfsHDgIU5AHl8PLo2Q3a%2Bv1VEu7vJvMg8BAL2yIJE%2FMQJ%2FDsCv4CIRcS3%2Fb4A9363xj3Buvr1voMFAUq1eQHHe3x8%2BDZHjQt7iXvChXyBf%2FXAu8d3%2F0P6PYX0iTe6fgFCfTZDO8A%2BusV4ecj%2BhMj2RUp3isA%2B%2FgBB%2FsL3BrcDPsB1%2F%2FtBgHoGvH7CyEB6gL7AwIZA%2F7j2N%2FXAwTv9wMY%2Fw4yAQgK5R3tJQPnEPjt8ebzKPYQ%2FAcRBdcVGSP98CYJFRQe7fniOCkNHhzlEAj9Gfv64iURJfj79v3y1BMS4vMM6gHu1BQBF%2FYeGOj4C%2FLM8g3%2FCurq9QoiIusR%2BRQn2hIH7fkk3%2Bk9BAIEFfES%2B8%2F50%2BcXIAsF%2B%2Brl8xIVAygH79QG7hg8%2FhXYJv3v%2Fwnm%2Bij%2F7%2F4qChbvBws48v7g%2Bewh6OPb3AYP4unmDckd3QvnBwfTKBwb7wX69doBBRnq1PjZBO79KjL7C%2FPe5wn%2FBOvg%2FycN1Bkl%2BwoGCPwO%2BvoK7OM3%2BA4h%2Bg7i0gH4DgTjARXX%2BwXaGhAH1hoe9f310dn9DvAQ7QPs2ODeCB8U4wT57ePyBwcd4NgE8%2FISC%2FkI6PARBCb3CBX9LOYl%2Fw%2FZDxI5De347ivd2P%2F%2FLQ7lwzLy8%2BnoFPrYAfLb8AITEy%2F36fvvzv7tGRQK0P448c%2F%2B4iwN9ggP8xkO%2BNce%2FRAxBgEgCR3kCN4l7PTr3esM6%2B7v9zPY4%2FYeJOY85iow9wjrChMC%2Fuzr%2B%2B3l%2BhcsCwoU5PP74wDp%2Bw3%2B9Rv2JB0LCvwXH%2FjwGdoUChsAG%2B3%2B5QsPAA0QNPHu5%2FgC%2BOT0EPL69iDv6wg0CQsF6v4X8%2B0oCPoa9gcW8fre9A%2Fx%2FQ8D8Ef91OgHBd4GBO0P7O4k2e%2Fy9%2F0F%2BTf28Pvv9v8yCdvyCAnmAPQZAewsEPLS2OAQ5eca1wsP7Nro7fzpARgI9vIe%2Bvz%2BBxDmGBH64vPw9SEAtefpFADn%2BPXRyykV8%2FkXBNre1RH97AUJFfTvCPUZDPXz7%2FLkDRQOHAoW%2BxAb7unZ4z4SGPkV2%2FPeFzwW5RLc7%2FpDCPzqAQ7qAAfq2Cj%2FHfDgHvLkDyUbDuPTKwIeE%2Fbs9AsBCATn%2BecH3e%2Fk4gPz9%2BwO%2FfgT%2B%2FH%2BCfkg%2FgLf9Pn8%2Fu3yPCn4Axb1%2B%2Fj%2B6ef1CxwTE90T7fnCBu3x9NgHKhbh5iT5%2Bfkh%2BPgNBw3w%2Bg3f%2BOodJQ%2F4BtfW5wEPMxns7vfvAALq%2Fv%2FaFAXYFxMoGfPp8REeGNMhR9Azsesllvs02MEWAx9Ef6z5B%2B3a6htJYzYU%2ByjxBuLw8SlLAAcT30HYP98EyRr4BSYHE%2FXb4%2B6Zq98CBRwGD6Uv8NHm6QsfIQ3tArQhG%2FMBWyIgu7kR1ujVpec9PzkrHfj23cUY%2F%2Bvr%2F%2Fq6H8880izr8CTW79XrAOPH7vYk5erpMPYKuwASCvEP4C%2F32%2BvO7woyNdsPdyH3DOEL68HqJNIM1vDo9AoJEQXwGikG5SLz4vvy1Cfn6v3f8yccFwz%2BFfzaw0XoFertDPwW9wFAJjLe3t29CerCARfo%2BTTIFOouxu8AYhj79jD4FxQBPPnw1sIb%2FyL1KQTTsvcL%2BwnTFicV7SvOFxzZNwwlDhwUAzXKFPRpDPIuvdP29NjP2fwe4VrpGAABIDANGP8xXEk9AB70LOAo0TXp7gIb%2Fvvl%2F%2BXTFPkaG%2BEwB%2FvQ0Qn019Xw6%2FzBEDESIMUT3fy0yeq6%2B%2BkG%2BwX0G7YW7%2B%2FZOgzkFu7a6eEO918n9M332R72IRUQ6xciKVHoDNdT%2BD8P8Aj13NL%2BKxUa1N0z%2BeTsHtQH49j737xV%2FODZprjr8%2F3d4iU9HikvFdw34tsGLiPR7tUr99MZ%2ByIL7%2FQKKEuy7MwJIPIM8uke1vTd3zw9LvoiEtwQ9wP4DwzxGQby09YU2gvEMO0C4hnH%2BgYT2xQ9xAoPPvpWCr4Byb%2Fd19HdYBkgS%2B%2B%2BAcn8Gc3qywj%2BYvpF3R%2Fp8dRAEeviCf%2F%2FGRYF87sa3OsDDxzy1ygdzDIbCUv1%2FPj9%2BwkbugjGD%2B5K6eABKc1cFSfcyAv%2FFhPzBOYW9vbWtzAC4wTr9urtB8nn5tAnBise5MEWEjM8K6EP7DvqBfLw6x%2FP7Ou89tEF2xX%2BLObuEwb38u0XJwtLE%2B8P6j3F9t7T9u%2FdU%2FwQDi8E%2BMobCSf63iPvA9EsRU4zP89VPEIK%2BBfjuzwIIAPcHc7W%2FhbFHVMsFe4CCQoZHwjCHAT34%2BU%2BF%2B0JLO8WJfD24L8P%2FOkIMdO5AxFQmRPdC0QUDSkG8vII%2Bh0KwuvOJ%2BrYEczyBua46PLczTH0uugU5d379QMKDgTlytTi%2FUDpGyreA%2F41AyIFReYI8gITuhLH8R0X5rkdzEXn5Ov6J%2BTywxIb4NkW4yYG9yUk7vOvE%2FAN%2Fxr7MQTc%2BPHNEiIFEhUBxvSzGRIaLFETFAZD2OwJ7%2F7uQQoiUvnp8A3gBdcRBljXEwA3%2BA4G5uje2du54wAF%2BfuzvBLRBNzbHP8S8goW8xUcExPr2fPOzFHr9%2BH7GwIn7QAj3CPDAwGvEhXvNgPOuggFBA%2FaCiD40iTlJucD6bnzHDzYBfkSJwv4%2BwEtJuRO7g8orgYK5O8v4wkX6xA639MACRPW%2BTMo%2Bd%2Bcu%2FvmOtvz5h8QLuTx%2BugV%2FAjvRn9gHAn56%2BXjCPUV7frs8fgIMSIVEgAb%2Bz4QCeP8SufrwiIJHNnaDv0eGPYL%2FAbqCvn19yMOE932%2FPnKGvDpywLr9O%2FrIAQV9PElGfv2Ov4E7uznEyHsCPUw7vsQAfwjBQzMBQb3GgUD9v0bHd%2Fe4hX%2B%2FuoI%2BSEF0icaAPwdIwQa4wALAPkNBeso%2BAjY6%2FbaAu7rMg8eERb%2BAxAC8RIIAg%2Fl9Tb4%2FP3%2B6ugS9yoRIhDmJiUU6xD62AAcBfwOIyHzBvQKzQoiEfMSGRUSLAX14f8SGbvhFhLk1hLg9wny5Qz05w8BBur%2F1wLw%2BvTe1w72LiYC7%2BvxCCr06NjdGRIDG%2Fzk%2BBba2uD6D%2FcT8BQP8e0T%2F%2BD5BfcZ%2FPED6kEJB98D7xIy1Or6%2Bw7KOwD24vECHPjk8wzS4x%2F95Nf%2B48bSCNzdFBQa0PDc3e7LGPYWIvbX3uwKLRX0BAEF3%2FXuHsYrISn9BOUPI%2FIVIOoK3gjj3PsE4t8KCBHkCuLq%2FRII5gEtGhrTLdgE9g3Z7gLgISUG5A%2FwABLI7uriBeQK7%2B%2FEIwDu0w%2Fu1gTW8ubq%2BiQfEEYTIg0I9iUy9w%2F6Cgzo5B8L7%2Bgg1tMcyeoa7NT%2F8R8RBu8U8x3xBA8M1s70OKT%2F0iU7KBbrIQcgFiYo9tDIx%2FP4%2FAz38Q75ISICOSMO9vUTCPcKHBAWA%2BL%2F6d3nBwsT7jbQ%2Bezw6Q%2Fi3wTd7xMI%2Bs886OX99Dg0IP0FCAMIPbwB1ijEE%2F316iEDE%2FvkEPn3BCINFwQXF9QH%2BAwm9RIj5xDWBwhH8hUSGwsS%2BSLz7iLaGiUICPPmLNwL8wwf5wnsC%2Fkk%2FPoD3xzxD%2BQG0OES2QXm3%2BX7FwoB%2Bfz%2B4ezUCv8M7AwyDQIDLOPtIQYFJ87yLvz%2F9gMIKDPo%2BjcG%2FBnuBgbRMwgEAiDOGw8b8xTjFA4J%2Fgjw0iMF%2ByvY1DUYzBbuEBPq4O8H%2FvUg%2Btnw8v4IIxYA8AgVFPDiC%2Fj9JRHn6%2F%2Fg1xsK8OgR5%2FXu9uwTLv8n0eoLKyP32PwF58kJvB4u9iIK3i8S8s4W4Rbm5UEV7fgEFwf%2BAPk4H%2BLaIgMmBMsCGd75Dfzw9v4z%2Bu7y9ve88BcTBtYMBSIj%2B9cTGfweB%2B%2FoKRb18%2FsbIQX0%2BQsW8gwc9MjoAvzyHAEnCgUGCCAMHgoOAisMJwQIDf39ABL%2FGPPyJyvw4vEI8i361RMB5xIR7Sn15icw%2FCfl4QYJQPkCF%2FLtDAnxDxUd0M3REQv5Aw377wbt3NzrQuwS%2Fg3t%2FOcqDPj%2F%2Fdr1vvwFzhLqOfvq3hUeMfsO9BUJ6fEQHtsrO%2BP16Qrr7%2BAL8x4G0eZN%2FuXO4hPt9CHL1ATyHEd%2F59rKGS8WAydxV9lA%2FQ0B3hTqAyn%2FIM%2FeINMn8yfu1dz%2BEg4gBQzt79AHrQ7gKvz04gwS1tgQ0%2BX33C4TAwkWHtgfzewH8t3Q6fn%2F8h%2F49%2FMd7Be29uDO3%2FYF7%2FMGCBb7NSHm58r58xQAwsza4QI6%2Fy4S%2Fl3hB%2F0xzvQ9FBcL%2Bf7pERz9wtxmDeskDwwS7BMs4OT%2BACmy2x3GJRsHAjgKBO%2FU6QC%2BFwj1IOL9PvMJDhbtEu7MIQse2ibz2%2B8H20ANBesM8NHwA%2BghBA43KRod5jXk6iUfFv3pNzUIx%2FnxIA4v%2FQz5LxQWJLfG7hv5Dvz53QXlAQACH7QoFPP7%2F%2FPwJ%2FnwChIM1xba6tIIw%2Frlxy0XN9cDMwM6IhFL%2FfwnGxsMyttN7PrMAu26Lv3j%2BNzu9wABABvu%2BiP86Qof4BXrtf3jCAL9A%2BkfxTf4Bu%2F55czv7RjNMhwQ1Sr3vT07Ex5A%2BvHWtsoPEx4hEfLZJygT5kHmB%2F0YHNTf7fbPHfjs9eW%2F4%2F7cDuwD5yT2Dv7%2B3SzWyiYTAhXP883f8tHJ9eDzRP%2FqFiD%2FTQjD6ihQDNfE3xjQBCUDMBbO%2BdPtDtUa7fsF5AcL5yrmyeXluwAd7Rb6%2BvLtJCYk7dza8hPlExvi%2FgUDMNbqQfvmHS8SHhDUIA44Ah4XxgIL2wj9C%2FUGEjrxEY7fCgo6JNuwCAgbETTZ2%2FnwBxYK%2FzNOCxHx5BQU2DYIB%2B3vRQ8N7g3QHCvA2hj4JkEmB9oA6Prc2AMJBf4K5A%2Fi0uLw99ow5%2Ff43drlGfYNUfHmLvT8Ag8x7O7w4QLCPPbuEtnYIUA0v9vcPPgpIucPAvAhCNH3%2BevZ9S8QCcIY9N36CPga0AAAAxMVJ%2Fch%2Fu3zAvkuHBjf9BDf%2FgP3Ogbp6NrXEQpBNvImDAktDwP3B%2F72GQUC7f0M9%2BAKCQk0RiHkHuT%2FE9DwPP0dMgIX%2B%2BEZ%2Fhf53i%2FzE%2BAAzfDsFv%2B8AeTw%2FyrDCOj8FR74Oxvz7gng7PgK9Ab%2FGLEd2e4d99QTA%2Bn9NhX7%2FSMHLfACDSHd9Lnz2tfwEt7m9w8bFvkKAQAUFgMF9kDNCOC1FQgG7w31Bvbu9grY%2FdYBB%2B3%2BCALZ9h0bGR4GFc0Z5R%2FkDgoHFun7yPTrHvYnIwoD1QIECeX3COzu2xkHFfIY4OoaEQQxJ%2BX3NBje3UAGKdUAKBYdEBr49fzcEBYG%2Bi03%2Fg%2Fn9v707AAKG%2FqV5SLv9yT5BiLeA8whHeHiDvi5CTQV%2FBDmLewP%2FtX08ds0CPvBqh0Z6ObxCtwBGwLsDengk%2FgLFf0Z7e%2FZAyf48TYKHxbk4vwE8xUEIv%2F3DiMF%2Fw3F3cU0DNLhPegGzdv3D8wd1df14g42Igzr%2BuMBUiYZf3be%2Bu0E%2F%2BM8WvPjCQsB6RM7Jtrs3wID6vQOIOQ7%2B%2F7h%2BfEc9yUDGxIM8yv56APHDw73OQP%2BFwf24fzw6bvDGTcoAfcJBw%2FaEtcJLhhCA%2BneQw7n4%2BLW803oExfp3PQA2gHQ9f%2F8RQLiyvzt8PEO9%2BoXDCH%2FBtvFFOTrH%2BQL%2BxH6DxgQ%2BBDa6fYU9%2BfWzBj96fgnFyL3F%2Bj%2BChEJIsT97ukHIdX3D%2Bbq5PvhDwPv9uUxKhskGu7pECos6AsB%2FvoT5gf5BRzu6B4I99wW3wMDE%2FO4NdsEL%2FT48gPh9Na9Me3P1g0M6NbE6gAu7fQWFf8y%2FxEG8hf0ADgNFcAoBQw0Hwzi%2BfL%2BDM4A9ibg1BbZIvIE3yI2APnc0v0XCRMq6BkM8egW0tAzEQ0AERcXA%2BPi5RT%2FBRkP9hLYr%2B4aAAzY9gHG6gTtCxjt4M4Y8Tn%2FHcoh8%2B8SFfcbOAIgDAMO%2Bw1EGg4dFDEN9fwi7NIc1Ab%2B6Ani4uwWH%2BIqK9wYFy33AtcQHAQU8u%2Fa9QAf6%2B8U6fjwL%2Fz53v%2FGAeEa4wj6Fvn02AwNAQvS%2BRLp6fUULRgTCyETEuvbKgn9%2Btm5B7sR9AHkyATdG%2FP%2BAB8J4CDqE%2Fc3%2BhTB8xH6%2BCvc%2BtNS7cLJDhtS4R0c9fQPQCfr%2FhDOH%2Fb66BL51OEBAgsNJfvZ%2FhXd9SID9hPi%2FvzL3v0CB%2F%2FnFdj51tYgBuXx6BTz%2FOIE%2Fw0I7ucj7B7UBxT7%2Buf49xs05OMV8hoTJ%2FbwCOnsAuqe5f%2Fzvw4HHgcmAh%2F38w4HB%2FjiI%2BDgCe0zFifVFQfyE%2BDr%2FQjryvn7BsH%2FCPYdAf%2F%2F5QYP8hTsAuvz%2FQLQBCP8984M%2FBLx1zH8D%2FHp8tUHDykMNe0bHQf8EQ4ayPfu%2BhM6CCjZFhALB%2BoMCfL17tH4yAcZEQQG9OoTG9jz8RDsFx7nHPUQAOv4HtsFChLmDNz2DAfF6OMj868hFu70%2Fwa6FCTRBxkH%2FOrsHxcV7%2F4W9vD8BwXu9yHqCOwbDSD5%2Bhoh9B8AHO8A%2BecO2PvYBxcZAfDL9%2F%2FXAfrQAe3xKAfb7%2Fb75fID6f0K8BYFBQjx7AoK89vu%2Fu8B9hcbHQf0FxrYA9b24fYIQgff%2ByTi9Pv%2B%2BbAm7OMR6vwS9wT69wrtMi0N9%2Fb17f32BtjnDzPVBDMY5PYG%2B%2BAN7vTlDhYW%2B%2BXu%2BcEL9h3xBv%2F02OkDD%2BIr7CDt9eoS%2BUTDBu3p3w7zDAHqFQ7YLeT4GjHv6OzzKAH9GfW7AAgF%2FRgN8xHk6%2FIzGQEW%2FRACDQUFLuwCCdcCsPwEKBcKGRr%2FBOcI6%2FHV9zr85uPt9e0Y%2FB4OCeLd%2BU3s8ujLL%2BjlAuPIEO3vQRUUG%2FznDDX37n8Q2y70JubrGPwc5yQE9wLqBxwG7AkTIwwLGhL%2FBAwE9w7%2BDvMSJvEBIQf52fDx6QsNEvEW%2BwAv%2B%2BQA9vb7Je%2FwC%2F73BxL1CAj%2BAOIE2A7w5xjt9wzg%2BAkuEA4dDC0B9O0i7PHz5irw%2B%2BYVAe8K%2Ffj4Le8Q3%2Bj69fgKAv3y5goQ9vn%2F9f8bDQEEAhH0%2BOH%2BDQMIGBT69wTv8QP4%2B%2FLsGw4S8ib1AQEBEQnnBvsMCh4C7A%2F%2B2PgIAQ4S6%2FgMHfgB8RwH6voC%2FBb9DOgB8N0OCAwZ6BcaJ%2B0bDvINDvP43AL4%2FvcD5fcB6R0U5PIEC%2FcL9BLnEAT9Gy30%2FxH1zukDJtX99wLeAAr07Pz1IMYD79zrMAbnFvj2%2BQr8%2BhPo9Rv7BPQJ8QDx0fYmFfkE%2F%2B8SEvYS%2BAEB7wP7BeEH%2BggLFR3zJPnf%2FSIJ1xPgDAQM%2BhOlDhks%2BR0s7%2FIG%2BwDX%2FPAFChECDg4T%2BAX86vUe8fb%2B5fwe7uj93wjiCPkNChgT%2FAsKCiQHKOoDCf%2F8JPX65gEE%2BRoL4v%2F7%2F%2FHm6Qrt2wYZG%2Bz0GSnu89z%2B0RUe%2FukYF%2FUG7%2Brj6vfsEO%2Ft6%2BnmFAcBEw8MEO0UExUB%2BgP9EfAc%2FfH56gIA%2BTn8CA3tGO%2F09wv9Fx0c5uAP%2F94RJfAw9wj%2F%2BfrjCgQXH%2FwX6ff%2BDRT8yAMJBxTn%2FAYMEO3rCRX4Afru9d7dDQn%2F7QkbGQL%2FJucd9QLn7gPmCf7nNCPsABkOxQHxEAwFCxcFK%2FUu%2Bvr06ukLJwXo6On89gcB3P0V%2B%2FgDFQ%2FuAyHs5gYe3ufXDQjd9v4V%2Bfrw3u4Z%2Fgw2%2Bej7CgYD%2BgsT%2BhX2ExXOC%2FUsDu0cGx%2F5%2FAkQDBjr%2BQscAxX7Bfn68Aj%2BESAPHOL47OrfDQ0BCw72Ewj89Qv04wMY9AnzDAECB%2BoNDhQn8wYB%2BQAj7RAO%2FQYD5zD2HPwK0O4M8f0K9AjsDwD9D%2FcA%2FesRBwUQ5gT8AfEB7Bnu6vf4G%2BcDB%2BEh8QoMFvIg5gskBOL4DhYB7Pf58%2FX87wYY%2Bh8YCBz%2F7Ar5Dhgf%2Fgvz7uoqBvfxGgYK6vT59u8HGAs4AQvxFQs%2B8AP6Gw%2F8BA0AD%2F%2FzHO7zDNMe8v0CFRb88A3rzgsBAgjvDPsAAwsL5%2FkG4QvzDwD%2F7PL97gD3Fgr87%2Fn2%2BAH5EOYI%2Bxn9DggI8Q7f7OsK4P4ZGu8IBg%2FbEuvaCSIFB9z4HSkXIwvjFs%2Fd9AcB0AcG%2FC70DvwL9QXsAzH75O%2Fy5gDm8hYE8wvvAeQMAvnwAyEQ%2Fen2%2FhQVJfoZ6uQMAxMB9fj5E%2B8O7wT06vz%2F9fD57e3mC%2FPtFAzgyeB%2FIAoE5uuz7AjYlvHbCzwSIwUcts4f3%2Bt5IfwXAAbg5wkRI%2FEDLesC5RsXCvjrBAHwDhj7AgYCEuff7eT%2FCicP2RT4LtbtEPIEGgYCDRXeGAMRFuryHBTd9z8H5g8dAP%2FaEOjV%2Bv8LDOML8%2B0IAdvtIBgQKv4lIAgBA8jb6gY53P%2F4DAoCEdQYAQUD5u4ZE9AAIf31E%2FvnGQsOIN4v9ejUB%2FgE%2BAUCKPEL%2FSEJENbf6PoQIALp%2BwbqCv4tEQ4DB%2F7VB%2FkCCOcI3d4H%2BwT7F%2Fr6Bd4y3xf53RYG%2Bw4KKPUNJAgPKsrsFfgBBAX6DRf9F%2FjRDDfwFfU4wvow7foQ5vEHAQgRFADoB%2F3t%2FQb8%2BgfvIAvU7tvqBvn%2B%2BvcB7xkV4P0RCuz9AALozDEPAf%2F45gIUG%2FPN7P3nGQLl%2FPTyEQMFMij9KP%2FtDhbvGPv%2B7P4WJ9QCAw4cDfYS0vjbDfQF5d8l%2BA8hBioW%2F%2BsKL%2FoeERjy%2BAMK9CvmKOor5RgsPvDl6vL3DQLo9PAIEOoUBCPY6RDw%2BOXtFO0H6vIh%2BO%2Fn7fMJESD8BP3w8uIeCQkU7gsC%2Buvr9Pkb6PLWHyYlAfMRBt%2F%2FAgwULhvvC%2FjWARTq%2FBoC%2BBHt3gv%2F%2F%2FDn%2Bw8mHO8AFAAeIxPx8gf7Hcbs%2FSMSDg7v3xoNChkA9P70BMcBEAoP6xgYFf8C4OnmAusPEh8SCwUFD%2BXw%2FPHh4x3o%2BfH3B%2FnR4B3zChIDGAsN5BQJHvnUGu3Y6wXV%2BwIY7ygM4%2F4VMRb%2B6gAMMd%2FQ%2BAvRAfwNHQTsLxL12ADaCgfeHObt%2FOkIGOgEE9cBH%2Br2%2BgIo7Q3xEt7u0Ojv7fUDRR8DCAz0KgEcEAP6%2BAEEFfwLBycJ5Oon8fHhCdn%2B2SgB7TPg%2Fwsf3R%2F%2BCAELB%2FQJEegXHOj5Bgjx8ATl5%2FcLFwkX8AEB7%2BRLDfT28%2FMJ8AAL8CXpCh8W6u4I9w3zDP7c7u8LJdcp7e4FAMgFAxgl%2BO0f9uoK6OEEEQ4DHv4w4%2BUC6%2FEAEgT11f7oEAz7AhP5DPwG%2BB349QAB%2BPDj5%2F0yz%2Fv8Dx8L%2FOPa7S8JGPAIAQAo8w8CGvUS8RjmDQQLBhUSy%2FrjEfoK%2BBUGDukaJAUf8dIqAAz79f4LBA%2Fz5AEA8xf2EPADAColA%2BDq9evjBtoEEgvw%2FvIh3%2BrpH8E38P70EgAUDeklGv0TBuAE4hzr6BkHBwsVHM0N%2F%2FDu6gkP%2BwstECoY5yge9fr12uMa%2FefPFh4PBAz3GPrnExwW5NIE8uoz5RH0DBvz4UEf6Avs9d7u5uQq4Q8SJv0HBQvh3gEC6BMLABQGCgnxFRoE6hEPAQgfGQ4fDhHtBuHiCPg%2B9%2FIc%2FvgD%2Bh4SCeHPTtXl1u4KFfMN6YESyzYO2u8hK6z3JvkMQRPbPhQa%2BfTwChsYECQwCvk%2BCBkP4%2BANGBwG6PEMARv1CRjvFAIP7AwaFALa3%2FDkJf8E5B3cHAEU%2BSge%2FTPJ8gcGIuch2egByv%2F28h3zEAP3Hyf7CfIA8Qr9FAL7Fw4E%2Btve0BLxEckG6SH%2F9wbR9yEK6w4CC%2Fnp6e8gAzz98jEO698C5Rvh1BbyNfsWDuwW5MX69wTnrNzPAREG%2Bt3mAxrxHQEbHQEoBRgizfIJ8P35EP349dMXBA%2FsEBTm6BchBxnt8PQS%2FBgou%2FIiEgr%2F%2BBcqLBUd7QEk8%2FcS5QLW79oS6%2FjS%2BAcVEv%2Fx%2BwDW1grt8gkP5DsP%2B%2FkOwu7L6S8Z%2Bw7PBPQLJQLoDQf38Pob1fEB8RHrEBPXCxa9Bb0SKgcA%2BRn4Cxr1DCMZ2w4Y7xcKARj%2B3DHYC%2Fv8I%2FoP9hEEJ9Mi2fMJKwMbDRQn%2F%2FRC%2FfXwDdrg%2BzQX8yf37OQh6PP0E%2B%2Fj7PrYQu8eBvYJBhkkBAkf2gMFG%2BwcCtkPHxQ5I%2FwJ%2F%2F%2Fb5c0uCw0YBQHL9PvECQMB7vYxDt7x67jj5%2Fcb7BwCJAntHBsaBC0AAhgPEer55%2BkS6Pru6Qfa9%2F8zAeDlDiT8FNsAFiz9%2FQEM%2BwkH9fkBCg8QC%2FDl5u8f%2Fzn32TcWAhMP7bIgQDkdI%2F0jBOX4JhH1%2F%2FoB8vrfAycHEgPx7BI2Ffoi9w3%2B4hHm%2Bh7qAd%2F689Y8Bsrp6ycNENb12gnq9QXmKPUc%2Fxfs8uoOEBvYAkEIL%2FDxBA8LARzx7gkHBvcE4QztBAv55gP25ALh4xYB9QALNtDqEQTY5QQtCw3x%2FxL%2FFRLl1R3xCf0X%2FAMO7%2F0Q%2BBU19e35FfYEDfkASPL4OPoI9gnyOO7sCz%2F59e4a%2Bwb68vzsEQjlEdEG7AQQ0ezz9TvTR%2FkQGD%2F9B9wb9Q%2FsBdna3fQg%2BjsZFSUS7uYXDPjtKirmK%2Fr1Ef8luhIDDBIABwP%2FAeMZ%2Fyrl8hHZ%2F%2F4RJ%2BIBCdff%2FAgE2ecbDBYVAPDz4wzh4w0I294A%2F%2B8YCA0h%2FwX1yN4E4fEp3A%2F68xg2HAUVBNb28uob%2FQ%2F%2F6%2B4J%2Fgz%2B9PLzCf8RBwnxFwwQ%2BBD9C%2BT2K87p3P3j8%2FgX4PsK3APU3NDZAgnp%2FvwS2wv98vr8Af7s6RUDB%2FMRARPxC%2FgM6dMDIhT%2FLQgB3AcS9g8KEsLE%2BCHw5DU%2BI8QaxTLoCQb%2B%2BAfT8xUFJu7y%2BhsGCRQFD%2Bf93P79AgLz%2F%2F005%2BDsLt0M8PbzGO8sIhIY7ODj6hcDGuLuDQb17%2B0p6RYDAPEI88jSAgYFvf%2FvEfPv9Ab3OPMUEwcnFeoy29zm8%2FjnFNr88SENBibWBEPU3wnUzuLs7uCB9ej%2BNAn4%2FyMU9QoIDTBF4tzeHR3oFiEg4fzUCBPsNP7SDgrWBwP%2BFwrp%2B7719S%2F%2BAffn%2BR7pFvccAeQXzOMEEPjm0gjcGQwJ9ywZ8e3o5DXiB%2B4KB%2BQQEAe9DDPrzEoYBvT44v8Y5Pcc6hbp%2BynoAOr0DPPdBv8f6wEMA%2B896fES9AEJyNrzJBfq8RP4A%2F0sCAz39ub19wn0CwAU%2B%2BAECR4GCOfs6A8SB%2FkP5wj87wf4BwUbGuoPBxDtDxz97RcQ6RgU%2Fu38%2FhDw7BsKIeT94935%2BQIQEA7v%2BiYCBQv2BPjyBQD4FPf2Fu0MDhS0BO4FDPcM%2FRAKBOHyvvnzJB7jFwkgBd4d%2FAX99REF%2BxH45RsM6%2BbPGAn7BwkdBvQH5%2FwIFxEa8OwJ%2F%2FkN%2BBEGA%2F0d4A4P8Rr6%2BAEH%2FgH0FxM1GPYM7P0A%2FOAK2Qsf%2BfIAE%2BwIIujS7fTwBukSCdMA1ObpAvXr%2FRTk%2BvwK8Pb0Cfj5AAoZCvfuE%2FQUGuri6QoT7Q4VOh3%2F%2B9PkMA0a%2F%2FX9Hxbqufb8DwQDBfoN8%2FP5zwrfDPfm7N8V8%2BAJ2Pbj%2FuTp9PEcEBj9B%2F78Bf4GCvUO%2BfIF8hXcFgL1CPPj%2FQQT%2BxEi%2BBv%2B%2FeUEGhsE8P8CBfYaGgT4A%2FwGGBrk5fMU4%2BoDLPsOGf%2FuDwrzDQMLAEPy%2Fvcj%2Ff0eGu0R1vcMCRQFAxb3MhYDKO70%2Bvru5gcGB%2BkKEBP8FAjzDe4HAOvvBfj34QPn8QXs%2BB8W9ub6F%2Bnq7OACFhQLFfwj8wP9Evz5JPzjGfry9%2FHf4xD8%2B9%2F%2F8vok5RYYFeYUAwER%2FuzGBxwEDfcB%2FfUZ2AAcBfMMBxPkDPoE4%2F4EDgD7HgUU7x0A%2FRTpFv%2F5FOXmEwX64%2BTkMREX%2FBIODNHh9%2B4NHgAu%2BPkP9Ozq9AYuIR4ABQztPd3eAg0C%2BP7s%2FdrkFu0KDQ8M3Sv2GxL%2BGRAmJuv91iL49Q%2F47gAL9PgIGQ0G%2FQoT8vTcAQ8aChjt9SP3EOz5%2BRQeIRD8DQvjF%2FQG2ubv9%2F30EAEGDwP77esDCPoNEv3TDRjHygH%2BGfwMFdoSFvEBAwQA%2FQX5Mhn73v7jEf3y%2Bxf38NMI%2FvsOExrw%2Fg0hFQct4RYLAA4THg4JJ%2F3vBvwiFwn7FevVDQ8BEx389BIV2AcBBgP7AgboBhICD%2FL82vDvAgXq7Pb%2BEBz4zwz5EecE7yDkGf8RGBz%2BAgz6BdX6BwAg%2Be8G8woh3BntNPcUKAX1B%2BTrCvvwART8Ay36KgMWF%2F0ZD%2FMZFP8pFB3kFwrd8OW%2BEu4n6Bvw3CXmBABDAd4E9%2F3IEvwOAgTuM%2BcV%2BwMA9fn1CBH5AfcTAfE8Fw%3D%3D%22%0A%20%20%20%20EMB_SCALES%20%3D%20%5B0.000978397%2C%200.000907992%2C%200.00113808%2C%200.00111761%2C%200.00136229%2C%200.000989774%2C%200.00101956%2C%200.00122669%2C%200.000833593%2C%200.00114878%2C%200.000968963%2C%200.00110959%2C%200.00151749%2C%200.00129223%2C%200.00118718%2C%200.00139872%5D%0A%20%20%20%20EMB_DIM%20%3D%201024%0A%0A%20%20%20%20_q%20%3D%20np.frombuffer(base64.b64decode(EMB_B64)%2C%20dtype%3Dnp.int8).reshape(len(EMB_WORDS)%2C%20EMB_DIM)%0A%20%20%20%20EMB_V%20%3D%20_q.astype(np.float32)%20*%20np.array(EMB_SCALES%2C%20dtype%3Dnp.float32)%5B%3A%2C%20None%5D%0A%20%20%20%20EMB_N%20%3D%20EMB_V%20%2F%20np.linalg.norm(EMB_V%2C%20axis%3D1%2C%20keepdims%3DTrue)%0A%20%20%20%20return%20EMB_GROUPS%2C%20EMB_LABELS_EN%2C%20EMB_N%2C%20EMB_V%2C%20EMB_WORDS%0A%0A%0A%40app.cell%0Adef%20_(EMB_GROUPS%2C%20EMB_LABELS_EN%2C%20EMB_V%2C%20np%2C%20plt)%3A%0A%20%20%20%20%23%20PCA%EF%BC%9A%E4%B8%AD%E5%BF%83%E5%8C%96%20%2B%20SVD%20%E5%8F%96%E5%89%8D%E5%85%A9%E5%80%8B%E4%B8%BB%E6%88%90%E5%88%86%EF%BC%88%E7%9C%9F%E7%9A%84%E7%AE%97%EF%BC%8C%E4%B8%8D%E6%98%AF%E7%A4%BA%E6%84%8F%E5%9C%96%EF%BC%89%0A%20%20%20%20_Xc%20%3D%20EMB_V%20-%20EMB_V.mean(axis%3D0)%0A%20%20%20%20_U%2C%20_S%2C%20_Vt%20%3D%20np.linalg.svd(_Xc%2C%20full_matrices%3DFalse)%0A%20%20%20%20_P%20%3D%20_Xc%20%40%20_Vt%5B%3A2%5D.T%0A%20%20%20%20_palette%20%3D%20%5B%22%234C72B0%22%2C%20%22%23DD8452%22%2C%20%22%2355A868%22%2C%20%22%238172B2%22%5D%0A%20%20%20%20_names%20%3D%20%5B%22animals%22%2C%20%22transport%22%2C%20%22food%22%2C%20%22emotions%22%5D%0A%20%20%20%20_fig%2C%20_ax%20%3D%20plt.subplots(figsize%3D(7.0%2C%205.2))%0A%20%20%20%20for%20_g%20in%20range(4)%3A%0A%20%20%20%20%20%20%20%20_idx%20%3D%20%5B_i%20for%20_i%2C%20_gg%20in%20enumerate(EMB_GROUPS)%20if%20_gg%20%3D%3D%20_g%5D%0A%20%20%20%20%20%20%20%20_ax.scatter(_P%5B_idx%2C%200%5D%2C%20_P%5B_idx%2C%201%5D%2C%20s%3D90%2C%20color%3D_palette%5B_g%5D%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20edgecolor%3D%22%231C2B33%22%2C%20linewidth%3D1.1%2C%20zorder%3D3%2C%20label%3D_names%5B_g%5D)%0A%20%20%20%20for%20_i%2C%20_lab%20in%20enumerate(EMB_LABELS_EN)%3A%0A%20%20%20%20%20%20%20%20_ax.annotate(_lab%2C%20(_P%5B_i%2C%200%5D%2C%20_P%5B_i%2C%201%5D)%2C%20xytext%3D(6%2C%205)%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20textcoords%3D%22offset%20points%22%2C%20fontsize%3D9%2C%20fontweight%3D%22bold%22)%0A%20%20%20%20_ax.set_title(%2216%20real%201024-d%20embeddings%2C%20PCA%20to%202-D%22)%0A%20%20%20%20_ax.set_xlabel(%22PC1%22)%0A%20%20%20%20_ax.set_ylabel(%22PC2%22)%0A%20%20%20%20_ax.grid(alpha%3D0.25)%0A%20%20%20%20_ax.legend(fontsize%3D9%2C%20loc%3D%22best%22)%0A%20%20%20%20_fig.tight_layout()%0A%20%20%20%20_fig%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%E6%83%85%E7%B7%92%E5%8F%A5%E8%87%AA%E6%88%90%E4%B8%80%E5%9C%8B%E3%80%81%E5%8B%95%E7%89%A9%E6%93%A0%E5%9C%A8%E4%B8%80%E8%B5%B7%E2%80%94%E2%80%94%E4%BD%86%E6%B3%A8%E6%84%8F%EF%BC%9A**2%20%E7%B6%AD%E5%9C%B0%E5%9C%96%E6%98%AF%E8%A2%AB%E5%A3%93%E6%89%81%E7%9A%84%E4%B8%96%E7%95%8C**%0A%20%20%20%20%EF%BC%881024%20%E7%B6%AD%E7%A1%AC%E5%A3%93%E5%88%B0%202%20%E7%B6%AD%EF%BC%8C%E4%BA%A4%E9%80%9A%E5%92%8C%E9%A3%9F%E7%89%A9%E5%B0%B1%E6%93%A0%E5%9C%A8%E4%B8%80%E5%A1%8A%E4%BA%86%EF%BC%89%E3%80%82%0A%20%20%20%20%E7%9C%9F%E6%AD%A3%E7%9A%84%E7%9B%B8%E4%BC%BC%E5%BA%A6%E8%A6%81%E5%9B%9E%E5%88%B0%E5%8E%9F%E5%A7%8B%E5%90%91%E9%87%8F%E7%AE%97%20**cosine%20%E7%9B%B8%E4%BC%BC%E5%BA%A6**%EF%BC%88%E5%A4%BE%E8%A7%92%E6%9C%89%E5%A4%9A%E5%B0%8F%EF%BC%89%E3%80%82%0A%20%20%20%20%E6%8C%91%E5%85%A9%E5%8F%A5%E6%AF%94%E6%AF%94%E7%9C%8B%EF%BC%8C%E5%86%8D%E7%9C%8B%E7%9C%8B%E8%AA%B0%E6%98%AF%E8%AA%B0%E7%9A%84%E6%9C%80%E8%BF%91%E9%84%B0%EF%BC%9A%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(EMB_WORDS%2C%20mo)%3A%0A%20%20%20%20pick_a%20%3D%20mo.ui.dropdown(options%3DEMB_WORDS%2C%20value%3DEMB_WORDS%5B0%5D%2C%20label%3D%22%E5%8F%A5%E5%AD%90%20A%22)%0A%20%20%20%20pick_b%20%3D%20mo.ui.dropdown(options%3DEMB_WORDS%2C%20value%3DEMB_WORDS%5B1%5D%2C%20label%3D%22%E5%8F%A5%E5%AD%90%20B%22)%0A%20%20%20%20mo.hstack(%5Bpick_a%2C%20pick_b%5D%2C%20justify%3D%22start%22%2C%20gap%3D2)%0A%20%20%20%20return%20pick_a%2C%20pick_b%0A%0A%0A%40app.cell%0Adef%20_(EMB_N%2C%20EMB_WORDS%2C%20mo%2C%20np%2C%20pick_a%2C%20pick_b)%3A%0A%20%20%20%20_ia%2C%20_ib%20%3D%20EMB_WORDS.index(pick_a.value)%2C%20EMB_WORDS.index(pick_b.value)%0A%20%20%20%20_cos%20%3D%20float(EMB_N%5B_ia%5D%20%40%20EMB_N%5B_ib%5D)%0A%20%20%20%20_sims%20%3D%20EMB_N%20%40%20EMB_N%5B_ia%5D%0A%20%20%20%20_order%20%3D%20np.argsort(-_sims)%0A%20%20%20%20_nn%20%3D%20%5B(EMB_WORDS%5B_j%5D%2C%20float(_sims%5B_j%5D))%20for%20_j%20in%20_order%20if%20_j%20!%3D%20_ia%5D%5B%3A3%5D%0A%20%20%20%20_nn_rows%20%3D%20%22%5Cn%22.join(f%22%20%20%20%20%7B_k%2B1%7D.%20%E3%80%8C%7B_w%7D%E3%80%8D%20cosine%20%3D%20%7B_s%3A.3f%7D%22%20for%20_k%2C%20(_w%2C%20_s)%20in%20enumerate(_nn))%0A%20%20%20%20_hint%20%3D%20(%22%E2%80%94%E2%80%94%E5%BE%88%E5%83%8F%EF%BC%81%22%20if%20_cos%20%3E%200.35%20else%20%22%E2%80%94%E2%80%94%E6%99%AE%E9%80%9A%E6%8E%A5%E8%BF%91%E3%80%82%22%20if%20_cos%20%3E%200.2%20else%20%22%E2%80%94%E2%80%94%E4%B8%8D%E5%A4%AA%E7%9B%B8%E5%B9%B2%E3%80%82%22)%0A%20%20%20%20mo.md(%0A%20%20%20%20%20%20%20%20f%22%22%22%0A%20%20%20%20%20%20%20%20**cosine(%E3%80%8C%7Bpick_a.value%7D%E3%80%8D%2C%20%E3%80%8C%7Bpick_b.value%7D%E3%80%8D)%20%3D%20%7B_cos%3A.3f%7D**%7B_hint%7D%0A%0A%20%20%20%20%20%20%20%20%E3%80%8C%7Bpick_a.value%7D%E3%80%8D%E7%9A%84%E6%9C%80%E8%BF%91%E9%84%B0%EF%BC%88%E5%85%A8%E8%AA%9E%E6%96%99%E6%8E%92%E5%BA%8F%EF%BC%89%EF%BC%9A%0A%0A%20%20%20%20%7B_nn_rows%7D%0A%0A%20%20%20%20%20%20%20%20%E9%80%99%E5%80%8B%E3%80%8C%E7%AE%97%E7%9B%B8%E4%BC%BC%E3%80%81%E6%8E%92%E5%90%8D%E6%AC%A1%E3%80%8D%E7%9A%84%E5%8B%95%E4%BD%9C%EF%BC%8C%E5%B0%B1%E6%98%AF%E7%AC%AC%207%20%E8%AA%B2%20RAG%20%E6%AA%A2%E7%B4%A2%E7%9A%84%E5%85%A8%E9%83%A8%E6%95%B8%E5%AD%B8%E3%80%82%0A%20%20%20%20%20%20%20%20%22%22%22%0A%20%20%20%20)%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%23%23%203%EF%B8%8F%E2%83%A3%20Context%20Window%EF%BC%9A%E6%A8%A1%E5%9E%8B%E7%9A%84%E5%B7%A5%E4%BD%9C%E8%A8%98%E6%86%B6%0A%0A%20%20%20%20%E6%A8%A1%E5%9E%8B%E4%B8%80%E6%AC%A1%E8%83%BD%E3%80%8C%E7%9C%8B%E5%88%B0%E3%80%8D%E7%9A%84%20token%20%E4%B8%8A%E9%99%90%E5%8F%AB**%E4%B8%8A%E4%B8%8B%E6%96%87%E7%AA%97%EF%BC%88context%20window%EF%BC%89**%E3%80%82%0A%20%20%20%20prompt%20%EF%BC%8B%20%E5%B0%8D%E8%A9%B1%E6%AD%B7%E5%8F%B2%20%EF%BC%8B%20%E5%A1%9E%E9%80%B2%E5%8E%BB%E7%9A%84%E6%96%87%E4%BB%B6%20%EF%BC%8B%20%E6%A8%A1%E5%9E%8B%E7%9A%84%E5%9B%9E%E7%AD%94%EF%BC%8C**%E5%85%A8%E9%83%A8**%E8%A6%81%E6%93%A0%E5%9C%A8%E9%80%99%E5%80%8B%E7%AA%97%E8%A3%A1%EF%BC%9B%0A%20%20%20%20%E8%B6%85%E9%81%8E%E7%9A%84%E9%83%A8%E5%88%86%E4%B8%8D%E6%98%AF%E3%80%8C%E8%A8%98%E6%80%A7%E8%AE%8A%E5%B7%AE%E3%80%8D%EF%BC%8C%E6%98%AF**%E6%A0%B9%E6%9C%AC%E6%B2%92%E8%A2%AB%E9%80%81%E9%80%B2%E6%A8%A1%E5%9E%8B**%E3%80%82%0A%0A%20%20%20%20%E6%8B%89%E4%B8%80%E4%B8%8B%E4%BD%A0%E8%A6%81%E5%A1%9E%E7%9A%84%E6%96%87%E4%BB%B6%E9%87%8F%EF%BC%8C%E7%9C%8B%E5%AE%83%E8%B7%9F%E5%B8%B8%E8%A6%8B%E6%A8%A1%E5%9E%8B%E7%9A%84%E7%AA%97%E6%9C%89%E5%A4%9A%E5%A4%A7%E5%B7%AE%E8%B7%9D%0A%20%20%20%20%EF%BC%88%E4%B8%AD%E6%96%87%20token%20%E4%BC%B0%E7%AE%97%E7%94%A8%E7%AC%AC%201%20%E7%AF%80%E7%9A%84%E5%AF%A6%E6%B8%AC%E5%88%80%E5%B7%A5%EF%BC%9A%E5%B9%B3%E5%9D%87%201%20%E5%AD%97%20%E2%89%88%201.14%20token%EF%BC%89%EF%BC%9A%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(mo)%3A%0A%20%20%20%20doc_pages%20%3D%20mo.ui.slider(start%3D1%2C%20stop%3D2000%2C%20step%3D1%2C%20value%3D40%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20label%3D%22%E4%BD%A0%E8%A6%81%E5%A1%9E%E9%80%B2%E5%8E%BB%E7%9A%84%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6%EF%BC%88%E9%A0%81%EF%BC%8C%E6%AF%8F%E9%A0%81%E7%B4%84%20600%20%E5%AD%97%EF%BC%89%22%2C%20show_value%3DTrue)%0A%20%20%20%20doc_pages%0A%20%20%20%20return%20(doc_pages%2C)%0A%0A%0A%40app.cell%0Adef%20_(doc_pages%2C%20np%2C%20plt)%3A%0A%20%20%20%20%23%20%E6%A8%A1%E5%9E%8B%E4%B8%8A%E4%B8%8B%E6%96%87%E7%AA%97%EF%BC%88%E5%85%AC%E9%96%8B%E8%A6%8F%E6%A0%BC%EF%BC%89%EF%BC%9B%E6%96%87%E4%BB%B6%20token%20%E4%BC%B0%E7%AE%97%EF%BC%9A600%20%E5%AD%97%2F%E9%A0%81%20%C3%97%201.14%20token%2F%E5%AD%97%EF%BC%88o200k%20%E5%AF%A6%E6%B8%AC%E6%AF%94%E7%8E%87%EF%BC%89%0A%20%20%20%20_models%20%3D%20%5B%22Llama%203%20(8K)%22%2C%20%22Qwen3%2032K%22%2C%20%22Llama%203.1%20(128K)%22%2C%20%22GPT-4o%20(128K)%22%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%22Claude%20Haiku%204.5%20(200K)%22%2C%20%22Claude%20Opus%205%20(1M)%22%5D%0A%20%20%20%20_windows%20%3D%20np.array(%5B8_192%2C%2032_768%2C%20131_072%2C%20128_000%2C%20200_000%2C%201_000_000%5D)%0A%20%20%20%20_doc_tokens%20%3D%20doc_pages.value%20*%20600%20*%201.14%0A%20%20%20%20_fits%20%3D%20_windows%20%3E%3D%20_doc_tokens%0A%20%20%20%20_colors%20%3D%20%5B%22%2355A868%22%20if%20_f%20else%20%22%23C44E52%22%20for%20_f%20in%20_fits%5D%0A%20%20%20%20_fig%2C%20_ax%20%3D%20plt.subplots(figsize%3D(7.2%2C%204.2))%0A%20%20%20%20_ax.barh(_models%2C%20_windows%2C%20color%3D_colors%2C%20edgecolor%3D%22%231C2B33%22%2C%20linewidth%3D1.1%2C%20zorder%3D3)%0A%20%20%20%20_ax.axvline(_doc_tokens%2C%20color%3D%22%231C2B33%22%2C%20linestyle%3D%22--%22%2C%20linewidth%3D2.0%2C%20zorder%3D4)%0A%20%20%20%20_ax.text(_doc_tokens%20*%201.06%2C%20-0.42%2C%20f%22your%20doc%20~%7B_doc_tokens%2F1000%3A.0f%7DK%20tokens%22%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20fontsize%3D10%2C%20fontweight%3D%22bold%22%2C%20color%3D%22%231C2B33%22)%0A%20%20%20%20_ax.set_xscale(%22log%22)%0A%20%20%20%20_ax.set_xlabel(%22context%20window%20(tokens%2C%20log%20scale)%22)%0A%20%20%20%20_ax.set_title(%22green%20%3D%20your%20document%20fits%2C%20red%20%3D%20it%20does%20not%22)%0A%20%20%20%20_ax.grid(axis%3D%22x%22%2C%20alpha%3D0.3)%0A%20%20%20%20_fig.tight_layout()%0A%20%20%20%20_fig%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%E5%85%A9%E4%BB%B6%E5%80%BC%E5%BE%97%E8%A8%98%E4%BD%8F%E7%9A%84%E4%BA%8B%EF%BC%9A%0A%0A%20%20%20%20-%20**%E7%AA%97%E8%B6%8A%E5%A4%A7%E4%B8%8D%E7%AD%89%E6%96%BC%E8%B6%8A%E8%81%B0%E6%98%8E**%E2%80%94%E2%80%94%E5%A1%9E%E4%B8%80%E5%A0%86%E7%84%A1%E9%97%9C%E5%85%A7%E5%AE%B9%E5%8F%8D%E8%80%8C%E7%A8%80%E9%87%8B%E9%87%8D%E9%BB%9E%EF%BC%88%E7%AC%AC%206%20%E8%AA%B2%E7%9A%84%20context%20engineering%0A%20%20%20%20%20%20%E5%B0%B1%E5%9C%A8%E7%AE%A1%E9%80%99%E4%BB%B6%E4%BA%8B%EF%BC%89%E3%80%82%0A%20%20%20%20-%20**%E7%AA%97%E8%B6%8A%E5%A4%A7%EF%BC%8C%E4%BB%A3%E5%83%B9%E8%B6%8A%E9%87%8D**%E2%80%94%E2%80%94%E4%B8%8A%E4%B8%8B%E6%96%87%E4%B8%80%E9%95%B7%EF%BC%8C%E6%8E%A8%E7%90%86%E6%99%82%E8%A6%81%E7%B6%AD%E8%AD%B7%E7%9A%84%20KV%20cache%20%E8%A8%98%E6%86%B6%E9%AB%94%E6%9A%B4%E5%A2%9E%0A%20%20%20%20%20%20%EF%BC%88%E9%80%99%E6%98%AF%E7%AC%AC%203%20%E8%AA%B2%E7%9A%84%E4%B8%BB%E7%B7%9A%EF%BC%89%E3%80%82%0A%0A%20%20%20%20%23%23%204%EF%B8%8F%E2%83%A3%20Autoregressive%EF%BC%9A%E4%B8%80%E6%AC%A1%E5%90%90%E4%B8%80%E5%80%8B%20token%0A%0A%20%20%20%20LLM%20%E7%94%9F%E6%88%90%E6%96%87%E5%AD%97%E7%9A%84%E6%96%B9%E5%BC%8F%E6%98%AF**%E8%87%AA%E8%BF%B4%E6%AD%B8**%EF%BC%9A%E7%9C%8B%E8%91%97%E5%89%8D%E9%9D%A2%E6%89%80%E6%9C%89%20token%EF%BC%8C%E7%AE%97%E5%87%BA%E3%80%8C%E4%B8%8B%E4%B8%80%E5%80%8B%20token%20%E7%9A%84%E6%A9%9F%E7%8E%87%E5%88%86%E5%B8%83%E3%80%8D%EF%BC%8C%0A%20%20%20%20%E6%8A%BD%E4%B8%80%E5%80%8B%E5%87%BA%E4%BE%86%EF%BC%8C%E6%8E%A5%E5%88%B0%E5%8F%A5%E5%B0%BE%EF%BC%8C%E5%86%8D%E7%AE%97%E4%B8%8B%E4%B8%80%E5%80%8B%E2%80%A6%E2%80%A6%E4%BD%A0%E7%9C%8B%E5%88%B0%E5%9B%9E%E7%AD%94%E4%B8%80%E5%80%8B%E5%AD%97%E4%B8%80%E5%80%8B%E5%AD%97%E8%B9%A6%E5%87%BA%E4%BE%86%EF%BC%8C%E4%B8%8D%E6%98%AF%E5%8B%95%E7%95%AB%E6%95%88%E6%9E%9C%EF%BC%8C%0A%20%20%20%20%E6%98%AF%E5%AE%83**%E7%9C%9F%E7%9A%84%E4%B8%80%E6%AC%A1%E5%8F%AA%E7%AE%97%E4%B8%80%E5%80%8B**%E3%80%82%0A%0A%20%20%20%20%E4%B8%8B%E9%9D%A2%E7%94%A8%2012%20%E5%8F%A5%E5%B0%8F%E8%AA%9E%E6%96%99%E8%A8%93%E7%B7%B4%E4%B8%80%E5%80%8B%E3%80%8C%E7%9C%8B%E5%89%8D%E4%B8%80%E5%80%8B%E5%AD%97%E3%80%81%E7%8C%9C%E4%B8%8B%E4%B8%80%E5%80%8B%E5%AD%97%E3%80%8D%E7%9A%84%E8%BF%B7%E4%BD%A0%E6%A8%A1%E5%9E%8B%0A%20%20%20%20%EF%BC%88%E7%9C%9F%E7%9A%84%E7%B5%B1%E8%A8%88%E3%80%81%E7%9C%9F%E7%9A%84%E6%8A%BD%E6%A8%A3%E2%80%94%E2%80%94%E7%95%B6%E7%84%B6%EF%BC%8C%E5%AE%83%E6%AF%94%20LLM%20%E7%AC%A8%E5%8D%81%E5%84%84%E5%80%8D%EF%BC%8C%E4%BD%86%E7%94%9F%E6%88%90%E8%BF%B4%E5%9C%88%E4%B8%80%E6%A8%A1%E4%B8%80%E6%A8%A3%EF%BC%89%EF%BC%9A%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(pairwise)%3A%0A%20%20%20%20%23%20%E8%BF%B7%E4%BD%A0%E8%87%AA%E8%BF%B4%E6%AD%B8%E6%A8%A1%E5%9E%8B%EF%BC%9A%E5%AD%97%E5%85%83%20bigram%20%E8%A8%88%E6%95%B8%EF%BC%88P(%E4%B8%8B%E4%B8%80%E5%AD%97%20%7C%20%E5%89%8D%E4%B8%80%E5%AD%97)%EF%BC%89%0A%20%20%20%20AR_CORPUS%20%3D%20%5B%0A%20%20%20%20%20%20%20%20%22%E4%BB%8A%E5%A4%A9%E5%A4%A9%E6%B0%A3%E5%BE%88%E5%A5%BD%22%2C%20%22%E4%BB%8A%E5%A4%A9%E5%A4%A9%E6%B0%A3%E4%B8%8D%E5%A5%BD%22%2C%20%22%E4%BB%8A%E5%A4%A9%E5%BF%83%E6%83%85%E5%BE%88%E5%A5%BD%22%2C%20%22%E6%98%8E%E5%A4%A9%E5%A4%A9%E6%B0%A3%E5%BE%88%E5%A5%BD%22%2C%0A%20%20%20%20%20%20%20%20%22%E6%88%91%E4%BB%8A%E5%A4%A9%E5%BE%88%E9%96%8B%E5%BF%83%22%2C%20%22%E6%88%91%E6%98%8E%E5%A4%A9%E8%A6%81%E4%B8%8A%E7%8F%AD%22%2C%20%22%E6%88%91%E4%BB%8A%E5%A4%A9%E8%A6%81%E4%B8%8A%E5%AD%B8%22%2C%20%22%E5%A4%A9%E6%B0%A3%E5%A5%BD%E5%B0%B1%E5%87%BA%E5%8E%BB%E7%8E%A9%22%2C%0A%20%20%20%20%20%20%20%20%22%E5%BF%83%E6%83%85%E4%B8%8D%E5%A5%BD%E5%B0%B1%E5%90%83%E8%9B%8B%E7%B3%95%22%2C%20%22%E6%88%91%E5%BE%88%E5%96%9C%E6%AD%A1%E5%90%83%E8%9B%8B%E7%B3%95%22%2C%20%22%E6%88%91%E5%BE%88%E5%96%9C%E6%AD%A1%E8%B2%93%22%2C%20%22%E8%B2%93%E5%BE%88%E5%8F%AF%E6%84%9B%22%2C%0A%20%20%20%20%5D%0A%20%20%20%20AR_COUNTS%20%3D%20%7B%7D%0A%20%20%20%20for%20_s%20in%20AR_CORPUS%3A%0A%20%20%20%20%20%20%20%20_seq%20%3D%20%22%5E%22%20%2B%20_s%20%2B%20%22%24%22%0A%20%20%20%20%20%20%20%20for%20_a%2C%20_b%20in%20pairwise(_seq)%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20AR_COUNTS.setdefault(_a%2C%20%7B%7D).setdefault(_b%2C%200)%0A%20%20%20%20%20%20%20%20%20%20%20%20AR_COUNTS%5B_a%5D%5B_b%5D%20%2B%3D%201%0A%20%20%20%20return%20(AR_COUNTS%2C)%0A%0A%0A%40app.cell%0Adef%20_(mo)%3A%0A%20%20%20%20ar_temp%20%3D%20mo.ui.slider(start%3D0.1%2C%20stop%3D2.0%2C%20step%3D0.1%2C%20value%3D1.0%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20label%3D%22temperature%EF%BC%88%E8%B6%8A%E4%BD%8E%E8%B6%8A%E4%BF%9D%E5%AE%88%E3%80%81%E8%B6%8A%E9%AB%98%E8%B6%8A%E7%8B%82%E9%87%8E%EF%BC%89%22%2C%20show_value%3DTrue)%0A%20%20%20%20ar_seed%20%3D%20mo.ui.slider(start%3D1%2C%20stop%3D30%2C%20step%3D1%2C%20value%3D1%2C%20label%3D%22%E6%8A%BD%E6%A8%A3%E6%89%B9%E6%AC%A1%EF%BC%88%E6%8B%89%E4%B8%80%E4%B8%8B%EF%BC%9D%E9%87%8D%E6%8A%BD%EF%BC%89%22%2C%20show_value%3DTrue)%0A%20%20%20%20mo.vstack(%5Bar_temp%2C%20ar_seed%5D)%0A%20%20%20%20return%20ar_seed%2C%20ar_temp%0A%0A%0A%40app.cell%0Adef%20_(AR_COUNTS%2C%20ar_seed%2C%20ar_temp%2C%20html_mod%2C%20mo%2C%20np)%3A%0A%20%20%20%20_rng%20%3D%20np.random.default_rng(ar_seed.value)%0A%20%20%20%20_steps%20%3D%20%5B%5D%0A%20%20%20%20_cur%20%3D%20%22%5E%22%0A%20%20%20%20for%20_t%20in%20range(20)%3A%0A%20%20%20%20%20%20%20%20_dist%20%3D%20AR_COUNTS.get(_cur)%0A%20%20%20%20%20%20%20%20if%20not%20_dist%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20break%0A%20%20%20%20%20%20%20%20_chars%20%3D%20list(_dist)%0A%20%20%20%20%20%20%20%20_w%20%3D%20np.array(%5B_dist%5B_c%5D%20for%20_c%20in%20_chars%5D%2C%20dtype%3Dfloat)%0A%20%20%20%20%20%20%20%20_p%20%3D%20_w%20**%20(1.0%20%2F%20ar_temp.value)%0A%20%20%20%20%20%20%20%20_p%20%3D%20_p%20%2F%20_p.sum()%0A%20%20%20%20%20%20%20%20_ch%20%3D%20str(_rng.choice(_chars%2C%20p%3D_p))%0A%20%20%20%20%20%20%20%20_steps.append((_cur%2C%20_chars%2C%20_p%2C%20_ch))%0A%20%20%20%20%20%20%20%20if%20_ch%20%3D%3D%20%22%24%22%3A%0A%20%20%20%20%20%20%20%20%20%20%20%20break%0A%20%20%20%20%20%20%20%20_cur%20%3D%20_ch%0A%0A%20%20%20%20_out%20%3D%20%22%22.join(_ch%20for%20_c%2C%20_cs%2C%20_p%2C%20_ch%20in%20_steps%20if%20_ch%20!%3D%20%22%24%22)%0A%20%20%20%20_cur2%2C%20_chars2%2C%20_p2%2C%20_pick2%20%3D%20_steps%5B-1%5D%0A%20%20%20%20_rows%20%3D%20%22%22.join(%0A%20%20%20%20%20%20%20%20f'%3Cdiv%20style%3D%22display%3Aflex%3Balign-items%3Acenter%3Bgap%3A8px%3Bmargin%3A3px%200%22%3E'%0A%20%20%20%20%20%20%20%20f'%3Cspan%20style%3D%22width%3A2.2em%3Bfont-weight%3A800%3Bfont-family%3Amonospace%22%3E'%0A%20%20%20%20%20%20%20%20f%22%7Bhtml_mod.escape('%E7%B5%90%E6%9D%9F'%20if%20_c%20%3D%3D%20'%24'%20else%20_c)%7D%3C%2Fspan%3E%22%0A%20%20%20%20%20%20%20%20f'%3Cdiv%20style%3D%22flex%3A1%3Bbackground%3A%23EEF1F4%3Bborder-radius%3A6px%3Bheight%3A18px%3Boverflow%3Ahidden%22%3E'%0A%20%20%20%20%20%20%20%20f'%3Cdiv%20style%3D%22width%3A%7B_pp*100%3A.1f%7D%25%3Bheight%3A100%25%3Bbackground%3A%7B%22%23DD8452%22%20if%20_c%20%3D%3D%20_pick2%20else%20%22%234C72B0%22%7D%22%3E%3C%2Fdiv%3E%3C%2Fdiv%3E'%0A%20%20%20%20%20%20%20%20f'%3Cspan%20style%3D%22font-family%3Amonospace%3Bfont-size%3A12px%3Bwidth%3A4em%22%3E%7B_pp%3A.2f%7D%3C%2Fspan%3E%3C%2Fdiv%3E'%0A%20%20%20%20%20%20%20%20for%20_c%2C%20_pp%20in%20sorted(zip(_chars2%2C%20_p2)%2C%20key%3Dlambda%20x%3A%20-x%5B1%5D)%0A%20%20%20%20)%0A%20%20%20%20mo.vstack(%5B%0A%20%20%20%20%20%20%20%20mo.md(f%22**%E7%94%9F%E6%88%90%E7%B5%90%E6%9E%9C%EF%BC%9A%E3%80%8C%7B_out%7D%E3%80%8D**%EF%BC%88%E5%BE%9E%E5%8F%A5%E9%A6%96%E8%A8%98%E8%99%9F%20%60%5E%60%20%E9%96%8B%E5%A7%8B%EF%BC%8C%E4%B8%80%E6%AC%A1%E6%8A%BD%E4%B8%80%E5%80%8B%E5%AD%97%EF%BC%8C%E6%8A%BD%E5%88%B0%E7%B5%90%E6%9D%9F%E8%A8%98%E8%99%9F%E5%81%9C%EF%BC%89%22)%2C%0A%20%20%20%20%20%20%20%20mo.md(f%22%E6%9C%80%E5%BE%8C%E4%B8%80%E6%AD%A5%EF%BC%9A%E7%9C%8B%E8%91%97%E3%80%8C%7B'%E5%8F%A5%E9%A6%96'%20if%20_cur2%20%3D%3D%20'%5E'%20else%20_cur2%7D%E3%80%8D%EF%BC%8C%E4%B8%8B%E4%B8%80%E5%80%8B%E5%AD%97%E7%9A%84%E6%A9%9F%E7%8E%87%E5%88%86%E5%B8%83%EF%BC%88%E6%A9%98%E8%89%B2%EF%BC%9D%E9%80%99%E6%AC%A1%E6%8A%BD%E4%B8%AD%E7%9A%84%EF%BC%89%EF%BC%9A%22)%2C%0A%20%20%20%20%20%20%20%20mo.Html(f%22%3Cdiv%20style%3D'max-width%3A420px'%3E%7B_rows%7D%3C%2Fdiv%3E%22)%2C%0A%20%20%20%20%5D)%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.md(r%22%22%22%0A%20%20%20%20%E5%A4%9A%E6%8B%89%E5%B9%BE%E6%AC%A1%E3%80%8C%E6%8A%BD%E6%A8%A3%E6%89%B9%E6%AC%A1%E3%80%8D%EF%BC%9A%E5%90%8C%E4%B8%80%E5%80%8B%E5%88%86%E5%B8%83%EF%BC%8C%E6%AF%8F%E6%AC%A1%E6%8A%BD%E5%87%BA%E4%BE%86%E7%9A%84%E5%8F%A5%E5%AD%90%E4%B8%8D%E4%B8%80%E6%A8%A3%E2%80%94%E2%80%94%E9%80%99%E5%B0%B1%E6%98%AF%E7%82%BA%E4%BB%80%E9%BA%BC%0A%20%20%20%20**%E5%90%8C%E4%B8%80%E5%80%8B%E5%95%8F%E9%A1%8C%20LLM%20%E6%AF%8F%E6%AC%A1%E7%AD%94%E5%BE%97%E4%B8%8D%E5%AE%8C%E5%85%A8%E4%B8%80%E6%A8%A3**%E3%80%82%E6%8A%8A%20temperature%20%E5%A3%93%E5%88%B0%200.1%EF%BC%8C%E5%AE%83%E5%B9%BE%E4%B9%8E%E5%8F%AA%E6%8C%91%E6%9C%80%E9%AB%98%E6%A9%9F%E7%8E%87%E7%9A%84%E5%AD%97%EF%BC%8C%0A%20%20%20%20%E8%BC%B8%E5%87%BA%E5%B0%B1%E8%AE%8A%E7%A9%A9%E5%AE%9A%EF%BC%88%E4%B9%9F%E8%AE%8A%E7%84%A1%E8%81%8A%EF%BC%89%E3%80%82%0A%0A%20%20%20%20%E4%B9%9F%E9%A0%86%E4%BE%BF%E7%9C%8B%E5%87%BA%E4%BA%86%E8%87%AA%E8%BF%B4%E6%AD%B8%E7%9A%84%E5%85%A9%E5%80%8B%E4%BB%A3%E5%83%B9%EF%BC%9A**%E4%B8%B2%E8%A1%8C**%EF%BC%88%E7%AC%AC%20N%20%E5%80%8B%E5%AD%97%E4%B8%80%E5%AE%9A%E8%A6%81%E7%AD%89%E7%AC%AC%20N%E2%88%921%20%E5%80%8B%E5%AD%97%EF%BC%89%E3%80%81%0A%20%20%20%20**%E9%87%8D%E8%A4%87%E8%A8%88%E7%AE%97**%EF%BC%88%E6%AF%8F%E4%B8%80%E6%AD%A5%E9%83%BD%E8%A6%81%E9%87%8D%E7%9C%8B%E6%95%B4%E6%AE%B5%E5%89%8D%E6%96%87%E2%80%94%E2%80%94%E6%89%80%E4%BB%A5%E6%89%8D%E9%9C%80%E8%A6%81%20KV%20cache%EF%BC%8C%E7%AC%AC%203%20%E8%AA%B2%E8%A6%8B%EF%BC%89%E3%80%82%0A%0A%20%20%20%20%23%23%205%EF%B8%8F%E2%83%A3%20%E4%BD%A0%E7%9A%84%E5%AF%A6%E9%A9%97%E5%8D%80%0A%0A%20%20%20%20%E4%B8%8B%E9%9D%A2%E9%80%99%E6%A0%BC%E6%98%AF%E4%BD%A0%E7%9A%84%EF%BC%8C%E6%94%B9%E5%AE%8C%E6%8C%89%20%E2%96%B6%20%E9%87%8D%E8%B7%91%E3%80%82%E6%8C%91%E6%88%B0%E5%9C%A8%E5%B7%A6%E9%A0%81%E3%80%8C%E6%8F%9B%E4%BD%A0%E5%8B%95%E6%89%8B%E3%80%8D%EF%BC%8C%E5%81%9A%E5%AE%8C%E5%86%8D%E9%96%8B%E8%A7%A3%E7%AD%94%E5%B0%8D%E7%85%A7%E3%80%82%0A%20%20%20%20%22%22%22)%0A%20%20%20%20return%0A%0A%0A%40app.cell%0Adef%20_(AR_COUNTS%2C%20BPE_CORPUS%2C%20bpe_segment%2C%20bpe_train)%3A%0A%20%20%20%20%23%20%3D%3D%3D%3D%3D%20%E4%BD%A0%E7%9A%84%E5%AF%A6%E9%A9%97%E5%8D%80%20%3D%3D%3D%3D%3D%0A%20%20%20%20%23%20LEVEL%201%20%E8%B5%B7%E9%BB%9E%EF%BC%9A%E6%94%B9%20probe%EF%BC%8C%E7%9C%8B%E4%B8%8D%E5%90%8C%E5%90%88%E4%BD%B5%E6%AC%A1%E6%95%B8%E4%B8%8B%E5%AE%83%E8%A2%AB%E5%88%87%E6%88%90%E5%B9%BE%E5%A1%8A%0A%20%20%20%20probe%20%3D%20%22newest%22%0A%20%20%20%20my_merges%2C%20_%20%3D%20bpe_train(BPE_CORPUS%2C%2010)%0A%20%20%20%20print(%22BPE%3A%22%2C%20probe%2C%20%22%E2%86%92%22%2C%20bpe_segment(probe%2C%20my_merges))%0A%0A%20%20%20%20%23%20LEVEL%202%20%E8%B5%B7%E9%BB%9E%EF%BC%9A%E8%AA%9E%E6%96%99%E8%A3%A1%E3%80%8C%E5%A4%A9%E3%80%8D%E5%BE%8C%E9%9D%A2%E6%8E%A5%E9%81%8E%E5%93%AA%E4%BA%9B%E5%AD%97%EF%BC%9F%0A%20%20%20%20print(%22P(%E4%B8%8B%E4%B8%80%E5%AD%97%20%7C%20%E5%A4%A9)%20%E7%9A%84%E8%A8%88%E6%95%B8%EF%BC%9A%22%2C%20AR_COUNTS.get(%22%E5%A4%A9%22))%0A%20%20%20%20return%0A%0A%0A%40app.cell(hide_code%3DTrue)%0Adef%20_(mo)%3A%0A%20%20%20%20mo.accordion(%0A%20%20%20%20%20%20%20%20%7B%0A%20%20%20%20%20%20%20%20%20%20%20%20%22%F0%9F%92%A1%20LEVEL%201%20%E5%8F%83%E8%80%83%E8%A7%A3%E7%AD%94%22%3A%20mo.md(%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20r%22%22%22%0A%20%20%20%20%E6%8A%8A%E7%AC%AC%201%20%E7%AF%80%E7%9A%84%E3%80%8C%E8%A6%81%E5%88%87%E7%9A%84%E8%A9%9E%E3%80%8D%E6%94%B9%E6%88%90%20%60wider%60%EF%BC%8C%E5%90%88%E4%BD%B5%E6%AC%A1%E6%95%B8%E6%8B%89%E5%88%B0%2010%EF%BC%9A%0A%0A%20%20%20%20-%20%60widest%60%20%E5%9C%A8%E8%AA%9E%E6%96%99%E5%87%BA%E7%8F%BE%E9%81%8E%20%E2%86%92%20%E5%AD%B8%E5%88%B0%20%60wi%60%E3%80%81%60widest%3C%2Fw%3E%60%20%E4%B9%8B%E9%A1%9E%E7%9A%84%E5%90%88%E4%BD%B5%20%E2%86%92%20%E5%88%87%E5%BE%97%E5%BE%88%E6%95%B4%0A%20%20%20%20-%20%60wider%60%20%E6%B2%92%E5%87%BA%E7%8F%BE%E9%81%8E%20%E2%86%92%20%E5%8F%AA%E8%83%BD%E7%94%A8%E5%AD%B8%E9%81%8E%E7%9A%84%E7%A2%8E%E5%A1%8A%E6%8B%BC%EF%BC%9A%60wi%60%20%2B%20%60d%60%20%2B%20%60e%60%20%2B%20%60r%3C%2Fw%3E%60%20%E4%B8%80%E9%A1%9E%E7%9A%84%E7%B5%84%E5%90%88%0A%0A%20%20%20%20%E9%80%99%E5%B0%B1%E6%98%AF%20BPE%20%E7%9A%84%E6%A0%B8%E5%BF%83%E6%80%A7%E6%A0%BC%EF%BC%9A**%E6%B2%92%E7%9C%8B%E9%81%8E%E7%9A%84%E8%A9%9E%E4%B8%8D%E6%9C%83%E7%82%B8%E6%8E%89%EF%BC%8C%E5%8F%AA%E6%98%AF%E5%88%87%E5%BE%97%E6%AF%94%E8%BC%83%E7%A2%8E**%EF%BC%88token%20%E6%95%B8%E8%AE%8A%E5%A4%9A%E3%80%81%E8%AE%8A%E8%B2%B4%EF%BC%89%E3%80%82%0A%20%20%20%20%E4%B8%AD%E6%96%87%E5%9C%A8%20%60o200k_base%60%20%E4%B8%8A%E5%B9%B3%E5%9D%87%201%20%E5%AD%97%20%E2%89%88%201.14%20token%EF%BC%8C%E5%B0%B1%E6%98%AF%E5%90%8C%E4%B8%80%E5%80%8B%E7%8F%BE%E8%B1%A1%E7%9A%84%E6%94%BE%E5%A4%A7%E7%89%88%E3%80%82%0A%20%20%20%20%22%22%22%0A%20%20%20%20%20%20%20%20%20%20%20%20)%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22%F0%9F%92%A1%20LEVEL%202%20%E5%8F%83%E8%80%83%E8%A7%A3%E7%AD%94%22%3A%20mo.md(%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20r%22%22%22%0A%20%20%20%20%E5%AF%A6%E9%A9%97%E5%8D%80%E5%8D%B0%E5%87%BA%E7%9A%84%E8%A8%88%E6%95%B8%EF%BC%9A%E3%80%8C%E5%A4%A9%E3%80%8D%E5%BE%8C%E9%9D%A2%E6%8E%A5%E9%81%8E%E3%80%8C%E6%B0%A3%E3%80%8D4%20%E6%AC%A1%E3%80%81%E3%80%8C%E5%A4%A9%E3%80%8D3%20%E6%AC%A1%E3%80%81%E3%80%8C%E8%A6%81%E3%80%8D2%20%E6%AC%A1%E3%80%81%0A%20%20%20%20%E3%80%8C%E5%BF%83%E3%80%8D%E3%80%8C%E5%BE%88%E3%80%8D%E5%90%84%201%20%E6%AC%A1%E2%80%94%E2%80%94%E6%9C%80%E5%B8%B8%E6%8E%A5%E7%9A%84%E6%98%AF%E3%80%8C%E6%B0%A3%E3%80%8D%E3%80%82%E6%8A%8A%20temperature%20%E5%A3%93%E5%88%B0%200.1%20%E4%B9%8B%E5%BE%8C%EF%BC%8C%0A%20%20%20%20%E6%AF%8F%E4%B8%80%E6%AD%A5%E5%B9%BE%E4%B9%8E%E9%83%BD%E6%8C%91%E6%9C%80%E9%AB%98%E6%A9%9F%E7%8E%87%E7%9A%84%E5%AD%97%EF%BC%8C%E6%89%80%E4%BB%A5%E7%94%9F%E6%88%90%E5%B9%BE%E4%B9%8E%E6%AF%8F%E6%AC%A1%E9%83%BD%E8%B5%B0%E3%80%8C%E5%A4%A9%E2%86%92%E6%B0%A3%E3%80%8D%E9%80%99%E6%A2%9D%E8%B7%AF%EF%BC%9B%0A%20%20%20%20%E6%8B%89%E5%88%B0%202.0%20%E4%B9%8B%E5%BE%8C%E4%BD%8E%E6%A9%9F%E7%8E%87%E7%9A%84%E5%AD%97%E4%B9%9F%E5%B8%B8%E8%A2%AB%E6%8A%BD%E4%B8%AD%EF%BC%8C%E5%8F%A5%E5%AD%90%E5%B0%B1%E9%96%8B%E5%A7%8B%E4%BA%82%E8%B5%B0%E3%80%82%0A%0A%20%20%20%20%E5%B0%8D%E6%87%89%E5%88%B0%E7%9C%9F%E7%9A%84%20LLM%EF%BC%9A%60temperature%3D0%60%20%E6%8B%BF%E4%BE%86%E8%A6%81%E7%A9%A9%E5%AE%9A%E8%BC%B8%E5%87%BA%EF%BC%88%E5%88%86%E9%A1%9E%E3%80%81%E6%8A%BD%E6%AC%84%E4%BD%8D%EF%BC%89%EF%BC%8C%0A%20%20%20%20%E9%AB%98%20temperature%20%E6%8B%BF%E4%BE%86%E8%A6%81%E5%A4%9A%E6%A8%A3%E6%80%A7%EF%BC%88brainstorm%E3%80%81%E5%AF%AB%E6%96%87%E6%A1%88%EF%BC%89%E3%80%82%0A%20%20%20%20%22%22%22%0A%20%20%20%20%20%20%20%20%20%20%20%20)%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22%F0%9F%92%A1%20LEVEL%203%20%E6%8F%90%E7%A4%BA%22%3A%20mo.md(%0A%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20%20r%22%22%22%0A%20%20%20%20%E5%9C%A8%E7%AC%AC%202%20%E7%AF%80%E6%8C%91%E3%80%8C%E4%B8%80%E6%9D%AF%E7%86%B1%E6%8B%BF%E9%90%B5%E5%92%96%E5%95%A1%E3%80%8D%EF%BC%8C%E7%9C%8B%E5%AE%83%E7%9A%84%E6%9C%80%E8%BF%91%E9%84%B0%E2%80%94%E2%80%94%E7%AC%AC%E4%B8%80%E5%90%8D%E6%98%AF%E3%80%8C%E4%B8%80%E7%A2%97%E7%86%B1%E9%A8%B0%E9%A8%B0%E7%9A%84%E6%8B%89%E9%BA%B5%E3%80%8D%0A%20%20%20%20%EF%BC%88cosine%200.423%EF%BC%8C%E9%81%A0%E9%81%A0%E7%94%A9%E9%96%8B%E5%85%B6%E4%BB%96%E5%8F%A5%E5%AD%90%EF%BC%89%EF%BC%8C%E8%80%8C%E5%90%8C%E5%B1%AC%E3%80%8C%E9%A3%9F%E7%89%A9%E7%B5%84%E3%80%8D%E7%9A%84%E3%80%8C%E4%B8%80%E5%A1%8A%E8%8D%89%E8%8E%93%E8%9B%8B%E7%B3%95%E3%80%8D%E5%8F%AA%E6%9C%89%200.143%EF%BC%8C%0A%20%20%20%20%E7%94%9A%E8%87%B3%E6%8E%92%E5%9C%A8%E3%80%8C%E4%B8%80%E9%9A%BB%E5%8F%AF%E6%84%9B%E7%9A%84%E8%B2%93%E3%80%8D%EF%BC%880.171%EF%BC%89%E5%BE%8C%E9%9D%A2%E3%80%82%0A%0A%20%20%20%20%E9%80%99%E4%B8%8D%E6%98%AF%20bug%EF%BC%9Aembedding%20%E6%A8%A1%E5%9E%8B%E7%9C%8B%E7%9A%84%E6%98%AF**%E6%95%B4%E5%8F%A5%E7%9A%84%E8%AA%9E%E6%84%8F%E7%B5%84%E5%90%88**%EF%BC%88%E3%80%8C%E7%86%B1%E3%80%8D%E3%80%8C%E4%B8%80%E6%9D%AF%EF%BC%8F%E4%B8%80%E7%A2%97%E3%80%8D%E9%80%99%E7%A8%AE%0A%20%20%20%20%E7%95%B6%E4%B8%8B%E6%83%85%E5%A2%83%E7%9A%84%E7%9B%B8%E4%BC%BC%EF%BC%89%EF%BC%8C%E4%B8%8D%E6%98%AF%E4%BD%A0%E5%BF%83%E8%A3%A1%E7%9A%84%E7%9B%AE%E9%8C%84%E5%88%86%E9%A1%9E%E6%A8%99%E7%B1%A4%E3%80%82%E6%88%91%E5%80%91%E7%95%AB%202D%20%E5%9C%B0%E5%9C%96%E6%99%82%E7%94%A8%E9%A1%8F%E8%89%B2%E6%8A%8A%E5%AE%83%E5%80%91%0A%20%20%20%20%E5%88%86%E6%88%90%E5%9B%9B%E7%B5%84%EF%BC%8C%E4%BD%86%E5%90%91%E9%87%8F%E7%A9%BA%E9%96%93%E8%87%AA%E5%B7%B1%E4%B8%8D%E7%9F%A5%E9%81%93%E6%9C%89%E9%80%99%E5%9B%9E%E4%BA%8B%E3%80%82%0A%0A%20%20%20%20%E5%B8%B6%E8%B5%B0%E7%9A%84%E5%88%A4%E6%96%B7%EF%BC%9A**RAG%20%E6%AA%A2%E7%B4%A2%E5%88%B0%E7%9A%84%E3%80%8C%E7%9B%B8%E8%BF%91%E3%80%8D%E6%98%AF%E8%AA%9E%E6%84%8F%E7%9B%B8%E8%BF%91%EF%BC%8C%E4%B8%8D%E6%98%AF%E7%9B%AE%E9%8C%84%E5%88%86%E9%A1%9E**%E2%80%94%E2%80%94%0A%20%20%20%20%E5%AF%AB%20chunk%20%E8%88%87%E6%9F%A5%E8%A9%A2%E7%9A%84%E6%8E%AA%E8%BE%AD%E6%9C%83%E7%9B%B4%E6%8E%A5%E5%BD%B1%E9%9F%BF%E6%AA%A2%E7%B4%A2%E5%93%81%E8%B3%AA%EF%BC%8C%E7%AC%AC%207%20%E8%AA%B2%E6%9C%83%E7%9C%8B%E5%88%B0%E9%80%99%E4%BB%B6%E4%BA%8B%E7%9A%84%E5%AF%A6%E9%9A%9B%E5%BE%8C%E6%9E%9C%E3%80%82%0A%20%20%20%20%22%22%22%0A%20%20%20%20%20%20%20%20%20%20%20%20)%2C%0A%20%20%20%20%20%20%20%20%7D%0A%20%20%20%20)%0A%20%20%20%20return%0A%0A%0Aif%20__name__%20%3D%3D%20%22__main__%22%3A%0A%20%20%20%20app.run()%0A