Skip to content

Commit

Permalink
bug fix
Browse files Browse the repository at this point in the history
  • Loading branch information
KoichiYasuoka committed Jul 4, 2024
1 parent bf64261 commit 469bacc
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions demo/2024-07-05/ja-swallow-upos.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"!pip install transformers accelerate deplacy\n",
"from transformers import AutoTokenizer,AutoModelForTokenClassification,TokenClassificationPipeline\n",
"tkz=AutoTokenizer.from_pretrained(\"tokyotech-llm/Swallow-MS-7b-v0.1\")\n",
"mdl=AutoModelForTokenClassification.from_pretrained(\"KoichiYasuoka/Swallow-MS-7b-char-upos",trust_remote_code=True,device_map=\"auto\")\n",
"mdl=AutoModelForTokenClassification.from_pretrained(\"KoichiYasuoka/Swallow-MS-7b-char-upos\",trust_remote_code=True,device_map=\"auto\")\n",
"class TCP(TokenClassificationPipeline):\n",
" def check_model_type(self,supported_models):\n",
" pass\n",
Expand All @@ -43,7 +43,7 @@
"cell_type":"code",
"metadata":{ "colab_type":"code" },
"source": [
"tkz=AutoTokenizer.from_pretrained(\"KoichiYasuoka/Swallow-MS-7b-char-upos")\n",
"tkz=AutoTokenizer.from_pretrained(\"KoichiYasuoka/Swallow-MS-7b-char-upos\")\n",
"nlp=TCP(model=mdl,tokenizer=tkz)\n",
"txt=\"どこの村でも作っていた。\"\n",
"doc=\"\\n\".join(\"\\t\".join([str(i+1),txt[t[\"start\"]:t[\"end\"]],\"_\",t[\"entity\"].split(\"|\")[0]]+[\"_\"]*5+[\"SpaceAfter=No\"]) for i,t in enumerate(nlp(txt)))+\"\\n\\n\"\n",
Expand Down

0 comments on commit 469bacc

Please sign in to comment.