ManaTTS_Dataset_Processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7d87759ae07c4210b105536ed8f1d7b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6967b2ba72414d348705e9a2163021f9",
              "IPY_MODEL_8e8a551ff24547bfacb8eab28d35b0d8",
              "IPY_MODEL_1fdc233270f94eab917aef9bc11d9d8f"
            ],
            "layout": "IPY_MODEL_ddc638b85c2049508c96af8799474a47"
          }
        },
        "6967b2ba72414d348705e9a2163021f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_953effc86fa64c3d8fdf0ea11bf3a044",
            "placeholder": "​",
            "style": "IPY_MODEL_b597cb1773964e7db459776ceac2c735",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "8e8a551ff24547bfacb8eab28d35b0d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e22b35759544791a5b5cbf40099c579",
            "max": 1262097815,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4bece1fac8f0403699647636a6ef49fc",
            "value": 1262097815
          }
        },
        "1fdc233270f94eab917aef9bc11d9d8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab07a1dac600426eb90d83bb57bab070",
            "placeholder": "​",
            "style": "IPY_MODEL_77e0d132932044f68dc896d5a6162367",
            "value": " 1.26G/1.26G [00:20&lt;00:00, 73.2MB/s]"
          }
        },
        "ddc638b85c2049508c96af8799474a47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "953effc86fa64c3d8fdf0ea11bf3a044": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b597cb1773964e7db459776ceac2c735": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3e22b35759544791a5b5cbf40099c579": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bece1fac8f0403699647636a6ef49fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ab07a1dac600426eb90d83bb57bab070": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77e0d132932044f68dc896d5a6162367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9d4decba24d042da940773f87336daa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_127849756d73496aa485054aaf509fc9",
              "IPY_MODEL_aae59bd3434b44c68319798cca4eeb98",
              "IPY_MODEL_fd181c6b96ec46c2b3f653de1c450d5e"
            ],
            "layout": "IPY_MODEL_9675951567eb4a199b7c3910ec73ca1a"
          }
        },
        "127849756d73496aa485054aaf509fc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_37a7b341e91640729046040b11da4d78",
            "placeholder": "​",
            "style": "IPY_MODEL_20c3ad6ff91642cf91f62ac4e13bc355",
            "value": "preprocessor_config.json: 100%"
          }
        },
        "aae59bd3434b44c68319798cca4eeb98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c6695c4370d544dea3cba08b0ad26e9f",
            "max": 214,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9dc9ebdaddb947b188aecff65ade9b0c",
            "value": 214
          }
        },
        "fd181c6b96ec46c2b3f653de1c450d5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bbe21b0610124b4296db413832b1a465",
            "placeholder": "​",
            "style": "IPY_MODEL_df79fab4ce714f969be2dc45cdfa606e",
            "value": " 214/214 [00:00&lt;00:00, 6.93kB/s]"
          }
        },
        "9675951567eb4a199b7c3910ec73ca1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37a7b341e91640729046040b11da4d78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20c3ad6ff91642cf91f62ac4e13bc355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6695c4370d544dea3cba08b0ad26e9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9dc9ebdaddb947b188aecff65ade9b0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bbe21b0610124b4296db413832b1a465": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df79fab4ce714f969be2dc45cdfa606e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "33033c8887e6496b886376004b3abfdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b6edfeaaf5c9494c9b2140c984948b96",
              "IPY_MODEL_6d6faa657d3a4b32ae2e8df6e20a7b99",
              "IPY_MODEL_8301ceb0f67f45abb2207f124e7c3beb"
            ],
            "layout": "IPY_MODEL_2a5c05f51cd841908f686bd7c156e617"
          }
        },
        "b6edfeaaf5c9494c9b2140c984948b96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78bcd54be525451aa8964f9ea6208579",
            "placeholder": "​",
            "style": "IPY_MODEL_22f7a43f2e1843e4baf235c25cfd597f",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "6d6faa657d3a4b32ae2e8df6e20a7b99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5296356123b74b93a5ef94610e0cbe40",
            "max": 307,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0ef0b9d6329441b6ad18934e0bb0c5e3",
            "value": 307
          }
        },
        "8301ceb0f67f45abb2207f124e7c3beb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d525af4089aa4dfe89f18dfc956cb4a5",
            "placeholder": "​",
            "style": "IPY_MODEL_56037e23fa384dac9bfde200eb1db9a9",
            "value": " 307/307 [00:00&lt;00:00, 12.0kB/s]"
          }
        },
        "2a5c05f51cd841908f686bd7c156e617": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78bcd54be525451aa8964f9ea6208579": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22f7a43f2e1843e4baf235c25cfd597f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5296356123b74b93a5ef94610e0cbe40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ef0b9d6329441b6ad18934e0bb0c5e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d525af4089aa4dfe89f18dfc956cb4a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56037e23fa384dac9bfde200eb1db9a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f4aa391be68847d9ae1e973b9b874d2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_97311be152ae4d628ba9d06c9b68cabd",
              "IPY_MODEL_4a1ea6ff4f1249f69840c59b4fcf4a5a",
              "IPY_MODEL_2e02094c31aa47d4905e1129cbd27624"
            ],
            "layout": "IPY_MODEL_063a7232736f4d369614f31e848f7531"
          }
        },
        "97311be152ae4d628ba9d06c9b68cabd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_613290fa65b646d9ba46eb4d55555045",
            "placeholder": "​",
            "style": "IPY_MODEL_c53b18a97d2b468788ba926cf7ac6fd4",
            "value": "config.json: 100%"
          }
        },
        "4a1ea6ff4f1249f69840c59b4fcf4a5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6702d401cb944e5aafdcdc6f7769dacb",
            "max": 1568,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_01a64c76277a443abed60cd534da6556",
            "value": 1568
          }
        },
        "2e02094c31aa47d4905e1129cbd27624": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c47233985c014504b3d317adb0b43dff",
            "placeholder": "​",
            "style": "IPY_MODEL_246b1b72531f45608a5d1457e4848468",
            "value": " 1.57k/1.57k [00:00&lt;00:00, 71.3kB/s]"
          }
        },
        "063a7232736f4d369614f31e848f7531": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "613290fa65b646d9ba46eb4d55555045": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c53b18a97d2b468788ba926cf7ac6fd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6702d401cb944e5aafdcdc6f7769dacb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01a64c76277a443abed60cd534da6556": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c47233985c014504b3d317adb0b43dff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "246b1b72531f45608a5d1457e4848468": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f27b8be3b1d4472d96a21971335b7b2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_365a95b6c49e41139f56283d3ab99d63",
              "IPY_MODEL_e85e8aaa8f6c487896d3e9a54adee1fb",
              "IPY_MODEL_bed074073ff04bc084c2001d54f7a57c"
            ],
            "layout": "IPY_MODEL_693e5102e8cc4f4ca17617c5cd005d6e"
          }
        },
        "365a95b6c49e41139f56283d3ab99d63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ad03933e04ac4b45b039e8c1e6fc00da",
            "placeholder": "​",
            "style": "IPY_MODEL_3377744cdcf44b8f868c4431008c69c1",
            "value": "vocab.json: 100%"
          }
        },
        "e85e8aaa8f6c487896d3e9a54adee1fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09f4eb7e45e343a6a1d61df8fdfc40a9",
            "max": 399,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b7b898050fa14359a7c22882d7aa2e4b",
            "value": 399
          }
        },
        "bed074073ff04bc084c2001d54f7a57c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6b48a22604e34601b3dbb39f83baabcb",
            "placeholder": "​",
            "style": "IPY_MODEL_5c993dc224864323875e215e92abb3d3",
            "value": " 399/399 [00:00&lt;00:00, 20.0kB/s]"
          }
        },
        "693e5102e8cc4f4ca17617c5cd005d6e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ad03933e04ac4b45b039e8c1e6fc00da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3377744cdcf44b8f868c4431008c69c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "09f4eb7e45e343a6a1d61df8fdfc40a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b7b898050fa14359a7c22882d7aa2e4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6b48a22604e34601b3dbb39f83baabcb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c993dc224864323875e215e92abb3d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0611b5b810d143dda03ac9d091a38817": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_26b9c548bd0e44a295eaca7d3c55cc8c",
              "IPY_MODEL_71f09ed0ce7f4c12b0939ae8c7c480df",
              "IPY_MODEL_79ee671362ce43b9aebe2c99e6f28b33"
            ],
            "layout": "IPY_MODEL_64c70b973cb44e4ea9d5f16988b7922c"
          }
        },
        "26b9c548bd0e44a295eaca7d3c55cc8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b54fbf79be8147b391e3d5748316a076",
            "placeholder": "​",
            "style": "IPY_MODEL_46f94d59206245fb9f11f672286070f0",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "71f09ed0ce7f4c12b0939ae8c7c480df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71a0d24e426a4be4865820e59a9145b0",
            "max": 85,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ef668c8e047948f6a993f4a1b36355ca",
            "value": 85
          }
        },
        "79ee671362ce43b9aebe2c99e6f28b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b04fe8c4a4854135a7b60a323849a489",
            "placeholder": "​",
            "style": "IPY_MODEL_c062bb6d60764337abf49170cd9105cc",
            "value": " 85.0/85.0 [00:00&lt;00:00, 5.15kB/s]"
          }
        },
        "64c70b973cb44e4ea9d5f16988b7922c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b54fbf79be8147b391e3d5748316a076": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46f94d59206245fb9f11f672286070f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "71a0d24e426a4be4865820e59a9145b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef668c8e047948f6a993f4a1b36355ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b04fe8c4a4854135a7b60a323849a489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c062bb6d60764337abf49170cd9105cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e22365646a27465588ed9db99b3c6369": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_44ffeb102ac248a79ed8a78e88e720a6",
              "IPY_MODEL_abdf2df3309b43d2b269b58e4e76ab35",
              "IPY_MODEL_63bff76331a54effb3f0e55b5bcc0a1c"
            ],
            "layout": "IPY_MODEL_c216bdeadf2a47d08ec45aa307b49407"
          }
        },
        "44ffeb102ac248a79ed8a78e88e720a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7ac9f9ed7a04cb48c170d5d8480f948",
            "placeholder": "​",
            "style": "IPY_MODEL_c3d748cbbf8a4074a91e48091053ee0a",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "abdf2df3309b43d2b269b58e4e76ab35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f494ce2d4d0488eb7b990b85792a102",
            "max": 1262097815,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4fe2a439cbea4c768ab08b0ebba50084",
            "value": 1262097815
          }
        },
        "63bff76331a54effb3f0e55b5bcc0a1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77ab59b3486f479783b0a28d018f09b8",
            "placeholder": "​",
            "style": "IPY_MODEL_8092bf7be17b4a96a3965b8326b2916c",
            "value": " 1.26G/1.26G [00:21&lt;00:00, 64.9MB/s]"
          }
        },
        "c216bdeadf2a47d08ec45aa307b49407": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7ac9f9ed7a04cb48c170d5d8480f948": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3d748cbbf8a4074a91e48091053ee0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f494ce2d4d0488eb7b990b85792a102": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fe2a439cbea4c768ab08b0ebba50084": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "77ab59b3486f479783b0a28d018f09b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8092bf7be17b4a96a3965b8326b2916c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "605bef355a4045138dcc4dd8cd7c0e7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e5ec0cbeecac4d9daae25bd3bda378b3",
              "IPY_MODEL_22886044c08642bcb82ce8a0903220ca",
              "IPY_MODEL_f57d3a435a544bb1b6964bf4b92ee608"
            ],
            "layout": "IPY_MODEL_a99b0591024d406ebe73c6dcab03b9c4"
          }
        },
        "e5ec0cbeecac4d9daae25bd3bda378b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1cf7c9639b59465fa9d46e0a786a952c",
            "placeholder": "​",
            "style": "IPY_MODEL_593efdaeb4ff403c813ed00a41c28674",
            "value": "model_config.yaml: 100%"
          }
        },
        "22886044c08642bcb82ce8a0903220ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_720944f73de24edf9c7a59a00351f4a9",
            "max": 953,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7cd7fe1df9734190b376d7297cabaeba",
            "value": 953
          }
        },
        "f57d3a435a544bb1b6964bf4b92ee608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5da4239df1264534b0f91b152dabd8a9",
            "placeholder": "​",
            "style": "IPY_MODEL_a5f057d15f8d48d985ae3e43c0a0ec0c",
            "value": " 953/953 [00:00&lt;00:00, 31.1kB/s]"
          }
        },
        "a99b0591024d406ebe73c6dcab03b9c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cf7c9639b59465fa9d46e0a786a952c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "593efdaeb4ff403c813ed00a41c28674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "720944f73de24edf9c7a59a00351f4a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7cd7fe1df9734190b376d7297cabaeba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5da4239df1264534b0f91b152dabd8a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5f057d15f8d48d985ae3e43c0a0ec0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e643830ab2f849b29f3e825dbea4b209": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ad5687828ed049979d7cb284476a1367",
              "IPY_MODEL_8adbe31e6c5b4bbe9799dc2578226b72",
              "IPY_MODEL_fd1efb1aa2394cdeab00764a294671c2"
            ],
            "layout": "IPY_MODEL_8c679574dcf9473087419d53ea203d88"
          }
        },
        "ad5687828ed049979d7cb284476a1367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_690d6e20d2cd484d8235ef497e626ea6",
            "placeholder": "​",
            "style": "IPY_MODEL_f884ea959bb9417e837da1b3f83df8b3",
            "value": "model.pt: 100%"
          }
        },
        "8adbe31e6c5b4bbe9799dc2578226b72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a9ee970b8ac411a9ce7c0bbb48519c5",
            "max": 967089089,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ee7b0784f1664115aa9d545e69c56e35",
            "value": 967089089
          }
        },
        "fd1efb1aa2394cdeab00764a294671c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b52ec593133441ea3a0e97c669b1040",
            "placeholder": "​",
            "style": "IPY_MODEL_50c44d9257324d8e992e4c1430c9b244",
            "value": " 967M/967M [00:18&lt;00:00, 47.5MB/s]"
          }
        },
        "8c679574dcf9473087419d53ea203d88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "690d6e20d2cd484d8235ef497e626ea6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f884ea959bb9417e837da1b3f83df8b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a9ee970b8ac411a9ce7c0bbb48519c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee7b0784f1664115aa9d545e69c56e35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8b52ec593133441ea3a0e97c669b1040": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50c44d9257324d8e992e4c1430c9b244": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f18380e717264bd3b48a0c0fffc000a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_383584857b5246359020c0073f747312",
              "IPY_MODEL_ceaadfce5ab145bbafdba74c53519d90",
              "IPY_MODEL_87b6cb5f2c734851b61fe7c85e379faa"
            ],
            "layout": "IPY_MODEL_7b66aa1aecc64501b5e4007661ed49af"
          }
        },
        "383584857b5246359020c0073f747312": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_548264c266814a5abf0d12d5915b713a",
            "placeholder": "​",
            "style": "IPY_MODEL_e9de7a80364a4cecb158c1a3575fc5ff",
            "value": "(…)eprocessor/feature_extractor_config.yaml: 100%"
          }
        },
        "ceaadfce5ab145bbafdba74c53519d90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4986183c8e09464cb5ef650633041763",
            "max": 225,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c00d0942c6c64158bb41820bc9ccb696",
            "value": 225
          }
        },
        "87b6cb5f2c734851b61fe7c85e379faa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bea2ea0bee044dc58c45f88461bf82d5",
            "placeholder": "​",
            "style": "IPY_MODEL_0327b88f29804fa380aebdf1729fe01d",
            "value": " 225/225 [00:00&lt;00:00, 9.72kB/s]"
          }
        },
        "7b66aa1aecc64501b5e4007661ed49af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "548264c266814a5abf0d12d5915b713a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9de7a80364a4cecb158c1a3575fc5ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4986183c8e09464cb5ef650633041763": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c00d0942c6c64158bb41820bc9ccb696": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bea2ea0bee044dc58c45f88461bf82d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0327b88f29804fa380aebdf1729fe01d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7eb13221c89545f0a313033040257409": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dcf4a328117c435990c19a55824fdc64",
              "IPY_MODEL_fd1aa8c3660547119262d3caefec0d56",
              "IPY_MODEL_f4ef179c9df847b989e9e30fef7e078a"
            ],
            "layout": "IPY_MODEL_ff5b54e5cbe8445b95759d984e4166d0"
          }
        },
        "dcf4a328117c435990c19a55824fdc64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62da430f2248422ebe8afa91df674c7c",
            "placeholder": "​",
            "style": "IPY_MODEL_dcc7cc933413442780e1567f2daeb564",
            "value": "preprocessor/tokenizer_config.yaml: 100%"
          }
        },
        "fd1aa8c3660547119262d3caefec0d56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5df02567bc59454585b34a19d85858c2",
            "max": 1837,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_38f12986180e457dac8f91d4d6d1873c",
            "value": 1837
          }
        },
        "f4ef179c9df847b989e9e30fef7e078a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5bd7cb633484e3f9286075f98690576",
            "placeholder": "​",
            "style": "IPY_MODEL_f3fe8befcdcb4848a3f62904c194e7d5",
            "value": " 1.84k/1.84k [00:00&lt;00:00, 87.5kB/s]"
          }
        },
        "ff5b54e5cbe8445b95759d984e4166d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62da430f2248422ebe8afa91df674c7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dcc7cc933413442780e1567f2daeb564": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5df02567bc59454585b34a19d85858c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38f12986180e457dac8f91d4d6d1873c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b5bd7cb633484e3f9286075f98690576": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3fe8befcdcb4848a3f62904c194e7d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14a70290de474da3a381559a91854887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_113a6f4f78fd47c1b3eda612089b6e95",
              "IPY_MODEL_2b2dc1c9828d4e70b5fbd5e9d3be2c14",
              "IPY_MODEL_dc3d153b0197411fa9cad9b60b76861f"
            ],
            "layout": "IPY_MODEL_e81e60e5ea544117acdc3bb1b7f21ad9"
          }
        },
        "113a6f4f78fd47c1b3eda612089b6e95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_01b3a7c1f32a43478e1d665c60192ef8",
            "placeholder": "​",
            "style": "IPY_MODEL_d1d72169688841eebf478519464dd321",
            "value": "preprocessor/tokenizer.json: 100%"
          }
        },
        "2b2dc1c9828d4e70b5fbd5e9d3be2c14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c168f5641824b7f9b7e7d88248c82f5",
            "max": 2204003,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8337feb0025c4c06b7be9d63d468a19a",
            "value": 2204003
          }
        },
        "dc3d153b0197411fa9cad9b60b76861f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_674ef00642e1447387fbe2aa93d12148",
            "placeholder": "​",
            "style": "IPY_MODEL_dc877490474446dda6b8646638e38055",
            "value": " 2.20M/2.20M [00:00&lt;00:00, 19.8MB/s]"
          }
        },
        "e81e60e5ea544117acdc3bb1b7f21ad9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01b3a7c1f32a43478e1d665c60192ef8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1d72169688841eebf478519464dd321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2c168f5641824b7f9b7e7d88248c82f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8337feb0025c4c06b7be9d63d468a19a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "674ef00642e1447387fbe2aa93d12148": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc877490474446dda6b8646638e38055": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# ManaTTS Dataset Processing\n",
        "This notebook provides the complete processing pipeline for ManaTTS (link to be updated).\n",
        "\n",
        "To run the pipeline, first execute the Environment Setup cells. Then, place the raw audio and text files in a directory named raw and run the remaining cells in sequence. Note that the original pipeline was not executed on Colab. To offer an executable demo on Colab's free account with limited memory, we have commented out two of the ASR models. The rest of the pipeline remains unchanged.\n",
        "\n"
      ],
      "metadata": {
        "id": "7pdm1GmUt6m9"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Environment Setup"
      ],
      "metadata": {
        "id": "qm2b1llYZMlY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install hazm  # Requires Restart"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sUOKF5SYZhgW",
        "outputId": "6c8bdae3-981c-4e8f-abb7-c6e8c8c6ecf6"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: hazm in /usr/local/lib/python3.10/dist-packages (0.10.0)\n",
            "Requirement already satisfied: fasttext-wheel<0.10.0,>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from hazm) (0.9.2)\n",
            "Requirement already satisfied: flashtext<3.0,>=2.7 in /usr/local/lib/python3.10/dist-packages (from hazm) (2.7)\n",
            "Requirement already satisfied: gensim<5.0.0,>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from hazm) (4.3.2)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from hazm) (3.8.1)\n",
            "Requirement already satisfied: numpy==1.24.3 in /usr/local/lib/python3.10/dist-packages (from hazm) (1.24.3)\n",
            "Requirement already satisfied: python-crfsuite<0.10.0,>=0.9.9 in /usr/local/lib/python3.10/dist-packages (from hazm) (0.9.10)\n",
            "Requirement already satisfied: scikit-learn<2.0.0,>=1.2.2 in /usr/local/lib/python3.10/dist-packages (from hazm) (1.2.2)\n",
            "Requirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.10/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm) (2.12.0)\n",
            "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm) (67.7.2)\n",
            "Requirement already satisfied: scipy>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim<5.0.0,>=4.3.1->hazm) (1.11.4)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim<5.0.0,>=4.3.1->hazm) (6.4.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (2024.5.15)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (4.66.4)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn<2.0.0,>=1.2.2->hazm) (3.5.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install hezar  # Requires Restart"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "UTozBx24XF-2",
        "outputId": "a958aff8-2d33-4b5c-a0ad-a1ebd927b852"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting hezar\n",
            "  Downloading hezar-0.38.4-py2.py3-none-any.whl (180 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/180.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m174.1/180.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m180.7/180.7 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate>=0.27.0 (from hezar)\n",
            "  Downloading accelerate-0.30.1-py3-none-any.whl (302 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.6/302.6 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets>=2.9.0 (from hezar)\n",
            "  Downloading datasets-2.19.2-py3-none-any.whl (542 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.1/542.1 kB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface_hub>=0.12.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (0.23.2)\n",
            "Collecting omegaconf>=2.3.0 (from hezar)\n",
            "  Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (1.2.2)\n",
            "Requirement already satisfied: tensorboard>=2.10.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (2.15.2)\n",
            "Requirement already satisfied: tokenizers>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (0.19.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (2.3.0+cu121)\n",
            "Requirement already satisfied: transformers>=4.28.0 in /usr/local/lib/python3.10/dist-packages (from hezar) (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.27.0->hezar) (1.24.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.27.0->hezar) (24.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.27.0->hezar) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.27.0->hezar) (6.0.1)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.27.0->hezar) (0.4.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (3.14.0)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets>=2.9.0->hezar)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (2.0.3)\n",
            "Collecting requests>=2.32.1 (from datasets>=2.9.0->hezar)\n",
            "  Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.9/64.9 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (4.66.4)\n",
            "Collecting xxhash (from datasets>=2.9.0->hezar)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m20.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets>=2.9.0->hezar)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.9.0->hezar) (3.9.5)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub>=0.12.0->hezar) (4.12.0)\n",
            "Collecting antlr4-python3-runtime==4.9.* (from omegaconf>=2.3.0->hezar)\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->hezar) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->hezar) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->hezar) (3.5.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (1.64.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (3.6)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (3.20.3)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (67.7.2)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (1.16.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard>=2.10.0->hezar) (3.0.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->hezar) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->hezar) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->hezar) (3.1.4)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.10.0->hezar)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->hezar) (2.3.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->hezar)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m63.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.28.0->hezar) (2024.5.15)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.9.0->hezar) (4.0.3)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.10.0->hezar) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.10.0->hezar) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.10.0->hezar) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard>=2.10.0->hezar) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.1->datasets>=2.9.0->hezar) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.1->datasets>=2.9.0->hezar) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.1->datasets>=2.9.0->hezar) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.1->datasets>=2.9.0->hezar) (2024.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard>=2.10.0->hezar) (2.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.9.0->hezar) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.9.0->hezar) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.9.0->hezar) (2024.1)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->hezar) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.10.0->hezar) (0.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard>=2.10.0->hezar) (3.2.2)\n",
            "Building wheels for collected packages: antlr4-python3-runtime\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=b599a0bf774fe2b8a906ead689ff4c32bba12406badfcda803894c321b4c5dda\n",
            "  Stored in directory: /root/.cache/pip/wheels/12/93/dd/1f6a127edc45659556564c5730f6d4e300888f4bca2d4c5a88\n",
            "Successfully built antlr4-python3-runtime\n",
            "Installing collected packages: antlr4-python3-runtime, xxhash, requests, omegaconf, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, dill, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, nvidia-cusolver-cu12, datasets, accelerate, hezar\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.31.0\n",
            "    Uninstalling requests-2.31.0:\n",
            "      Successfully uninstalled requests-2.31.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests==2.31.0, but you have requests 2.32.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed accelerate-0.30.1 antlr4-python3-runtime-4.9.3 datasets-2.19.2 dill-0.3.8 hezar-0.38.4 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 omegaconf-2.3.0 requests-2.32.3 xxhash-3.4.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "pydevd_plugins",
                  "requests"
                ]
              },
              "id": "acd4b4550ef840268660f2c7c6c606d8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install pydub"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFSxcaOjW1ij",
        "outputId": "30605f1d-0480-4989-c7e8-f4d3bfd71f6b"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (0.25.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install pyaudioconvert"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e-vGzy6MWPSn",
        "outputId": "ff98f534-c32e-41fb-f9f2-ba8d653e5084"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyaudioconvert\n",
            "  Downloading pyaudioconvert-0.0.5-py3-none-any.whl (4.8 kB)\n",
            "Installing collected packages: pyaudioconvert\n",
            "Successfully installed pyaudioconvert-0.0.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install jiwer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cAsZgQnfYkgK",
        "outputId": "77eb7049-c871-4f4b-bd32-f5beaf71a0d6"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting jiwer\n",
            "  Downloading jiwer-3.0.4-py3-none-any.whl (21 kB)\n",
            "Requirement already satisfied: click<9.0.0,>=8.1.3 in /usr/local/lib/python3.10/dist-packages (from jiwer) (8.1.7)\n",
            "Collecting rapidfuzz<4,>=3 (from jiwer)\n",
            "  Downloading rapidfuzz-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m34.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: rapidfuzz, jiwer\n",
            "Successfully installed jiwer-3.0.4 rapidfuzz-3.9.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setup parsi io"
      ],
      "metadata": {
        "id": "NVlJhS9_cHaJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! git clone https://github.com/language-ml/parsi.io.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lrPGOeS6Zz-b",
        "outputId": "744e914f-8f10-4ea5-e290-83cc0c5de0bd"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'parsi.io'...\n",
            "remote: Enumerating objects: 1312, done.\u001b[K\n",
            "remote: Counting objects: 100% (1312/1312), done.\u001b[K\n",
            "remote: Compressing objects: 100% (738/738), done.\u001b[K\n",
            "remote: Total 1312 (delta 520), reused 1253 (delta 475), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (1312/1312), 62.25 MiB | 22.58 MiB/s, done.\n",
            "Resolving deltas: 100% (520/520), done.\n",
            "Updating files: 100% (217/217), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mv parsi.io parsi_io"
      ],
      "metadata": {
        "id": "eEROzSSgaAUK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "744bdc80-d06b-4dbc-b04d-e12a38e29db7"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mv: cannot move 'parsi.io' to 'parsi_io/parsi.io': Directory not empty\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IUxylP7I9-l_"
      },
      "source": [
        "## Setup Spleeter\n",
        "The audio source separation tool"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Jwcpn7oP13r",
        "outputId": "395ac7d5-65d8-413d-e686-d9cb4c063845"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'spleeter'...\n",
            "remote: Enumerating objects: 2670, done.\u001b[K\n",
            "remote: Counting objects: 100% (533/533), done.\u001b[K\n",
            "remote: Compressing objects: 100% (105/105), done.\u001b[K\n",
            "remote: Total 2670 (delta 460), reused 428 (delta 428), pack-reused 2137\u001b[K\n",
            "Receiving objects: 100% (2670/2670), 9.38 MiB | 26.31 MiB/s, done.\n",
            "Resolving deltas: 100% (1719/1719), done.\n"
          ]
        }
      ],
      "source": [
        "! git clone https://github.com/deezer/spleeter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 81,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "URViqxx5QLdU",
        "outputId": "a1fbe6a8-6f9d-45fd-dbc2-0b724b8357f3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: typer in /usr/local/lib/python3.10/dist-packages (0.9.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer) (8.1.7)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from typer) (4.12.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.24.3)\n",
            "Requirement already satisfied: tensorflow>=2.0.0 in /usr/local/lib/python3.10/dist-packages (2.15.0)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (24.3.25)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (0.5.4)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (0.2.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (3.9.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (18.1.1)\n",
            "Requirement already satisfied: ml-dtypes~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (0.2.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.24.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (24.0)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (67.7.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (4.12.0)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.14.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (0.37.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (1.64.0)\n",
            "Requirement already satisfied: tensorboard<2.16,>=2.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (2.15.2)\n",
            "Requirement already satisfied: tensorflow-estimator<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (2.15.0)\n",
            "Requirement already satisfied: keras<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.0.0) (2.15.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow>=2.0.0) (0.43.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (3.6)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (2.32.3)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (3.0.3)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (2024.2.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (2.1.5)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (0.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow>=2.0.0) (3.2.2)\n",
            "Requirement already satisfied: librosa in /usr/local/lib/python3.10/dist-packages (0.10.2.post1)\n",
            "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa) (3.0.1)\n",
            "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.24.3)\n",
            "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.11.4)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.2.2)\n",
            "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.4.2)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (4.4.2)\n",
            "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.58.1)\n",
            "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.12.1)\n",
            "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.8.1)\n",
            "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.3.7)\n",
            "Requirement already satisfied: typing-extensions>=4.1.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (4.12.0)\n",
            "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.4)\n",
            "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lazy-loader>=0.1->librosa) (24.0)\n",
            "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa) (0.41.1)\n",
            "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa) (4.2.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa) (2.32.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa) (3.5.0)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa) (1.16.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa) (2.22)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2024.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.0.3)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from pandas) (1.24.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
            "Collecting ffmpeg-python\n",
            "  Downloading ffmpeg_python-0.2.0-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from ffmpeg-python) (0.18.3)\n",
            "Installing collected packages: ffmpeg-python\n",
            "Successfully installed ffmpeg-python-0.2.0\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "ffmpeg is already the newest version (7:4.4.2-0ubuntu0.22.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 45 not upgraded.\n",
            "Collecting httpx[http2]\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx[http2]) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx[http2]) (2024.2.2)\n",
            "Collecting httpcore==1.* (from httpx[http2])\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx[http2]) (3.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx[http2]) (1.3.1)\n",
            "Collecting h2<5,>=3 (from httpx[http2])\n",
            "  Downloading h2-4.1.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.5/57.5 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx[http2])\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hyperframe<7,>=6.0 (from h2<5,>=3->httpx[http2])\n",
            "  Downloading hyperframe-6.0.1-py3-none-any.whl (12 kB)\n",
            "Collecting hpack<5,>=4.0 (from h2<5,>=3->httpx[http2])\n",
            "  Downloading hpack-4.0.0-py3-none-any.whl (32 kB)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx[http2]) (1.2.1)\n",
            "Installing collected packages: hyperframe, hpack, h11, httpcore, h2, httpx\n",
            "Successfully installed h11-0.14.0 h2-4.1.0 hpack-4.0.0 httpcore-1.0.5 httpx-0.27.0 hyperframe-6.0.1\n"
          ]
        }
      ],
      "source": [
        "! pip3 install typer\n",
        "! pip3 install numpy\n",
        "! pip3 install \"tensorflow>=2.0.0\"\n",
        "! pip3 install librosa\n",
        "! pip3 install pandas\n",
        "! pip3 install ffmpeg-python\n",
        "! sudo apt install -y ffmpeg\n",
        "! pip3 install httpx[http2]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 82,
      "metadata": {
        "id": "PsfCUSZEQnC4"
      },
      "outputs": [],
      "source": [
        "! mkdir -p spleeter/pretrained_models/2stems"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 83,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NhDDQThOQvfV",
        "outputId": "0fd6bfe1-8403-44b1-cf44-65fbbed0277e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-06-03 23:23:06--  https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz\n",
            "Resolving github.com (github.com)... 140.82.112.4\n",
            "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/211124697/e5a4d280-f98d-11e9-905c-849465861ed7?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20240603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240603T232306Z&X-Amz-Expires=300&X-Amz-Signature=c224f0c89a341c83be0764d4e995db59f2e50f18a65bd32bf3f0c5b99230fad8&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=211124697&response-content-disposition=attachment%3B%20filename%3D2stems.tar.gz&response-content-type=application%2Foctet-stream [following]\n",
            "--2024-06-03 23:23:06--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/211124697/e5a4d280-f98d-11e9-905c-849465861ed7?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=releaseassetproduction%2F20240603%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240603T232306Z&X-Amz-Expires=300&X-Amz-Signature=c224f0c89a341c83be0764d4e995db59f2e50f18a65bd32bf3f0c5b99230fad8&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=211124697&response-content-disposition=attachment%3B%20filename%3D2stems.tar.gz&response-content-type=application%2Foctet-stream\n",
            "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\n",
            "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 73109797 (70M) [application/octet-stream]\n",
            "Saving to: ‘2stems.tar.gz’\n",
            "\n",
            "2stems.tar.gz       100%[===================>]  69.72M   230MB/s    in 0.3s    \n",
            "\n",
            "2024-06-03 23:23:07 (230 MB/s) - ‘2stems.tar.gz’ saved [73109797/73109797]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "! wget https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz\n",
        "! mv 2stems.tar.gz spleeter/pretrained_models/2stems/2stems.tar.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 84,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YdboCr3qRAUI",
        "outputId": "e9f84ecb-f3b8-4ae0-8a47-35653eb940e9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "./._checkpoint\n",
            "checkpoint\n",
            "model.data-00000-of-00001\n",
            "model.index\n",
            "model.meta\n"
          ]
        }
      ],
      "source": [
        "! tar xvzf spleeter/pretrained_models/2stems/2stems.tar.gz -C spleeter/pretrained_models/2stems/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aHPcu4PN7O9D",
        "outputId": "cc701fdd-b553-4671-e746-41004a53fa00"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.24.3)\n",
            "Collecting numpy\n",
            "  Downloading numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m34.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: numpy\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.24.3\n",
            "    Uninstalling numpy-1.24.3:\n",
            "      Successfully uninstalled numpy-1.24.3\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "hazm 0.10.0 requires numpy==1.24.3, but you have numpy 1.26.4 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed numpy-1.26.4\n"
          ]
        }
      ],
      "source": [
        "! pip install numpy --upgrade"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Process Text"
      ],
      "metadata": {
        "id": "DN0jTR5XFVMm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "from functools import reduce\n",
        "from hazm import Normalizer\n",
        "from parsi_io.parsi_io.modules.number_extractor import NumberExtractor\n",
        "from parsi_io.parsi_io.modules.convert_number_to_text import ConvertNumberToText"
      ],
      "metadata": {
        "id": "8qt1qFc3OR4a"
      },
      "execution_count": 139,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Normalization"
      ],
      "metadata": {
        "id": "EjNPpBz1yhoi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "normalizer = Normalizer()\n",
        "\n",
        "def normalize_text(text):\n",
        "  return normalizer.normalize(text)"
      ],
      "metadata": {
        "id": "MupcUcpbytQq"
      },
      "execution_count": 140,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Symbol Substitution\n",
        "This step is designed to unify various forms of symbols into their more commonly used counterparts."
      ],
      "metadata": {
        "id": "ard6ph0Uy0dP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "substitution_dict = {'ﯽ': 'ی', '—': '–', '\\u200f': '\\u200c', '\\xad': '\\u200c', '\\u200e': '\\u200c', '\\u200d': '\\u200c'}\n",
        "\n",
        "def substitute_symbols(text):\n",
        "    translation_table = str.maketrans(substitution_dict)\n",
        "    substituted_text = text.translate(translation_table)\n",
        "    return substituted_text"
      ],
      "metadata": {
        "id": "fNfrmnVgzQV3"
      },
      "execution_count": 141,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove In-text References\n",
        "This step is designed to remove the references that come inside the text but are not read aloud. For example:\n",
        "> They have introduced a new tool [1] which ..."
      ],
      "metadata": {
        "id": "rfTD3b0nz4Lr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_inline_references(text):\n",
        "    # Define pattern to match references like \"[NUM]\"\n",
        "    pattern_fa = r\"\\[\\d+\\]|\\[۰-۹]+\\]\"\n",
        "    pattern_en = r\"\\[\\d+\\]|\\[0-9]+\\]\"\n",
        "\n",
        "    # Use regular expression to remove references\n",
        "    text_without_refs_fa = re.sub(pattern_fa, \" \", text)\n",
        "    text_without_refs_en = re.sub(pattern_en, \" \", text_without_refs_fa)\n",
        "\n",
        "    return text_without_refs_en"
      ],
      "metadata": {
        "id": "Ftu0Vajm0NQZ"
      },
      "execution_count": 142,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Reference Lines\n",
        "This step is designed to remove the references that come at the end of the text but are not read aloud. For example:\n",
        "> [1] Roshan-AI. Hazm. https://www.roshan-ai.ir/hazm/docs/index.html. Accessed:\n",
        "May 3, 2024.\n",
        ">\n",
        "> [2] ...\n",
        "\n"
      ],
      "metadata": {
        "id": "01MnXk8g0Qbc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_references_lines(text):\n",
        "    # Define pattern to match references like \"[NUM] \"\n",
        "    pattern_fa = r\"^\\s*\\[\\d+\\]|\\[۰-۹]+\\]\"\n",
        "    pattern_en = r\"^\\s*\\[\\d+\\]|\\[0-9]+\\]\"\n",
        "\n",
        "    # Split text into lines\n",
        "    lines = text.split('\\n')\n",
        "\n",
        "    # Remove lines starting with references\n",
        "    cleaned_lines = [line for line in lines if not re.match(pattern_fa, line.strip()) and not re.match(pattern_en, line.strip())]\n",
        "\n",
        "    # Join cleaned lines back into text\n",
        "    cleaned_text = '\\n'.join(cleaned_lines)\n",
        "\n",
        "    return cleaned_text"
      ],
      "metadata": {
        "id": "L7yk3Ube0nU_"
      },
      "execution_count": 143,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Link Lines\n",
        "This step is designed to remove the links and urls that come at the end of the text but are not read aloud. For example:\n",
        "> Resources:\n",
        ">\n",
        "> https://www.roshan-ai.ir/hazm/docs/index.html\n",
        ">\n",
        "> https://virgool.io/\n",
        ">\n",
        "> ..."
      ],
      "metadata": {
        "id": "9-jcYsWn0t9n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_link_lines(text):\n",
        "    # Define the pattern to match lines starting with http or www\n",
        "    pattern = r\"^\\s*(?:http|www)\"\n",
        "\n",
        "    # Split text into lines\n",
        "    lines = text.split('\\n')\n",
        "\n",
        "    # Remove lines starting with link\n",
        "    cleaned_lines = [line for line in lines if not re.match(pattern, line.strip())]\n",
        "\n",
        "    # Join cleaned lines back into text\n",
        "    cleaned_text = '\\n'.join(cleaned_lines)\n",
        "\n",
        "    return cleaned_text"
      ],
      "metadata": {
        "id": "U41eap2C0oWn"
      },
      "execution_count": 144,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Convert Numbers to Text\n",
        "This step is designed to convert the numbers in digit format into their spoken version. For example:\n",
        "\n",
        "> 22 → twenty two\n",
        "\n",
        "Converting numbers to their spoken format would be as simple as the `replace_numbers_with_text_normally` function. But the [parsi.io](https://github.com/language-ml/parsi.io) library has a bug reported [here](https://github.com/language-ml/parsi.io/issues/50). It actually cannot process text with a specific numeric phrases. Following are some examples of these phrases:\n",
        "\n",
        "> قرن سوم و چهارم هجری\n",
        ">\n",
        "> کلاس سوم و چهارم\n",
        ">\n",
        "> نوزده و بیست میلادی\n",
        "\n",
        "We have detected such phrases in the dataset by try-catching the error cases and handle them separately."
      ],
      "metadata": {
        "id": "08-yP9Hb1OV9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "persian_digits_pattern = re.compile(r'[۰۱۲۳۴۵۶۷۸۹0123456789]')\n",
        "num2text = ConvertNumberToText()\n",
        "extractor = NumberExtractor()"
      ],
      "metadata": {
        "id": "ZcVnmGRx1oBS"
      },
      "execution_count": 145,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def replace_numbers_with_text_at_exception(text):\n",
        "  reg = r'(سوم و چهارم|نوزده و بیست)'\n",
        "  # Split the text based on problematic expressions\n",
        "  splits = re.split(reg, text)\n",
        "  # Process each split individually\n",
        "  processed_splits = [replace_numbers_with_text(split) for split in splits if not re.search(reg, split)]\n",
        "  # Concatenate the processed splits\n",
        "  return ''.join(processed_splits)"
      ],
      "metadata": {
        "id": "epeVqKp9HE26"
      },
      "execution_count": 146,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def replace_numbers_with_text_normally(text):\n",
        "  # Find all number spans in the text\n",
        "  number_spans = extractor.run(text)\n",
        "\n",
        "  # Filter out spans that contain digits\n",
        "  filtered_spans = [span for span in number_spans if persian_digits_pattern.search(span['phrase'])]\n",
        "\n",
        "  # Convert the filtered numbers to text and replace them in the text\n",
        "  offset = 0  # Track the offset due to previous replacements\n",
        "  for span in filtered_spans:\n",
        "      start, end = span['span']\n",
        "      start -= offset  # Adjust start position based on previous replacements\n",
        "      end -= offset  # Adjust end position based on previous replacements\n",
        "      number_text = span['phrase']\n",
        "      number_value = span['value']\n",
        "\n",
        "      # Convert the number to text\n",
        "      text_value = num2text.run(str(number_value))\n",
        "\n",
        "      # Replace the number in the text with its textual equivalent\n",
        "      text = text[:start] + text_value + text[end:]\n",
        "\n",
        "      # Update the offset\n",
        "      offset += len(number_text) - len(text_value)\n",
        "\n",
        "  return text"
      ],
      "metadata": {
        "id": "HylkKIWp1saj"
      },
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def replace_numbers_with_text(text):\n",
        "    # The number extraction library has a bug reported here: https://github.com/language-ml/parsi.io/issues/50\n",
        "    # And example of this error happens on 'قرن سوم و چهارم هجری', 'کلاس سوم و چهارم', 'نوزده و بیست میلادی\n",
        "    # We will process these texts separately\n",
        "    try:\n",
        "        number_spans = extractor.run(text)\n",
        "        return replace_numbers_with_text_normally(text)\n",
        "    except (ValueError, IndexError) as e:\n",
        "        return replace_numbers_with_text_at_exception(text)"
      ],
      "metadata": {
        "id": "MvolHc_7GFKM"
      },
      "execution_count": 148,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Symbols\n",
        "This step is designed to remove some of the symbols that are not very common or do not affect the TTS-ASR models' outputs. This helps simplify the input to the models."
      ],
      "metadata": {
        "id": "Zyghd-a62tid"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "symbols_to_remove = \"«»*[]\\\"'^&<>{}|٫《》•\\x9d\\u200b\\x7f\"\n",
        "\n",
        "def remove_symbols(text):\n",
        "    pattern = \"[\" + re.escape(symbols_to_remove) + \"]\"\n",
        "    return re.sub(pattern, ' ', text)"
      ],
      "metadata": {
        "id": "VlA-vsW-29Vq"
      },
      "execution_count": 149,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Extra White Spaces\n",
        "This step is designed to remove extra white spaces inluding multiple consequent white spaces and new lines."
      ],
      "metadata": {
        "id": "AGc1r6vx3EdV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_extra_white_spaces(text):\n",
        "    cleaned_text = re.sub(r'\\s+', ' ', text)\n",
        "    return cleaned_text.strip()"
      ],
      "metadata": {
        "id": "DnJGXdap3Zc0"
      },
      "execution_count": 150,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Full Pipeline\n",
        "Here we define the complete text processing pipelin and the processing code."
      ],
      "metadata": {
        "id": "XwjdlP2o3Zya"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline = [\n",
        "    normalize_text,\n",
        "    substitute_symbols,\n",
        "    remove_inline_references,\n",
        "    remove_references_lines,\n",
        "    remove_link_lines,\n",
        "    replace_numbers_with_text,\n",
        "    remove_symbols,\n",
        "    remove_extra_white_spaces\n",
        "  ]"
      ],
      "metadata": {
        "id": "t9JzXgxM3hQO"
      },
      "execution_count": 151,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def process_text(input_file_path, output_dir_path):\n",
        "  input_file_name = input_file_path.split('/')[-1].split('.')[0]\n",
        "  output_file_path = os.path.join(output_dir_path, input_file_name + '.txt')\n",
        "\n",
        "  # Check if the output file already exists\n",
        "  if os.path.exists(output_file_path):\n",
        "    print(f\"Skipping file {input_file_name}.txt. Processed text file already exists.\")\n",
        "    return output_file_path\n",
        "\n",
        "  # Apply the text processing pipeline\n",
        "  with open(input_file_path, 'r') as f:\n",
        "    text = reduce(lambda txt, func: func(txt), pipeline, f.read())\n",
        "\n",
        "  # Export the processed text\n",
        "  with open(output_file_path, 'w') as f:\n",
        "     f.write(text)\n",
        "\n",
        "  return output_file_path"
      ],
      "metadata": {
        "id": "2HM4ETq_3kom"
      },
      "execution_count": 152,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Convert MP3 format to WAV\n",
        "MP3 format is lossy, meaning each time an audio file is encoded in MP3 format, it undergoes some quality degradation. To preserve the original quality of audio files during the processing pipeline, we convert them to the lossless WAV format."
      ],
      "metadata": {
        "id": "2m5HB5aBHVi6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from pydub import AudioSegment"
      ],
      "metadata": {
        "id": "1zNEDsayOYXN"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_mp3_to_wav(input_file_path, output_dir_path):\n",
        "    # Get the input file name without the extension\n",
        "    input_file_name = os.path.splitext(os.path.basename(input_file_path))[0]\n",
        "\n",
        "    # Construct the output file path\n",
        "    output_file_path = os.path.join(output_dir_path, f\"{input_file_name}.wav\")\n",
        "\n",
        "    # Load the MP3 file\n",
        "    audio = AudioSegment.from_mp3(input_file_path)\n",
        "\n",
        "    # Export the audio as a WAV file\n",
        "    audio.export(output_file_path, format=\"wav\")\n",
        "\n",
        "    return output_file_path"
      ],
      "metadata": {
        "id": "npSVeHzLHZLm"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Convert Stereo to Mono\n",
        "Converting audio from stereo to mono for a TTS dataset ensures consistency, simplifies processing, reduces storage needs, and aligns with the design of most TTS models, which are optimized for mono input. Mono audio eliminates unnecessary spatial effects, providing clear and intelligible speech essential for TTS applications."
      ],
      "metadata": {
        "id": "9fIXpcysIRtM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from pydub import AudioSegment"
      ],
      "metadata": {
        "id": "yyVEJ1zXOZ7W"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_stereo_to_mono(input_file_path, output_dir_path):\n",
        "    # Get the input file name with the extension\n",
        "    input_file_name = os.path.basename(input_file_path)\n",
        "\n",
        "    # Construct the output file path\n",
        "    output_file_path = os.path.join(output_dir_path, input_file_name)\n",
        "\n",
        "    # Load the audio file\n",
        "    audio = AudioSegment.from_wav(input_file_path)\n",
        "\n",
        "    # Convert stereo audio to mono\n",
        "    mono_audio = audio.set_channels(1)\n",
        "\n",
        "    # Save the mono audio file\n",
        "    mono_audio.export(output_file_path, format=\"wav\")\n",
        "\n",
        "    return output_file_path"
      ],
      "metadata": {
        "id": "BZRSXVUrIVeW"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Remove Background Music\n",
        "To have a clean speech file, it is essential to remove any potential background music. We use the source separation tool [spleeter](https://github.com/deezer/spleeter) for this purpose.\n",
        "\n",
        "In the following code, we split the audio into chunks of at most 10 minutes that can effectively fo under background music removal process and then concat back the processed chunks into the resulting audio file."
      ],
      "metadata": {
        "id": "uJyKcQ97IXHb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import subprocess\n",
        "import shutil\n",
        "from pydub import AudioSegment"
      ],
      "metadata": {
        "id": "hMx--w_VOdhK"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_file_name(full_path):\n",
        "    file_name_with_extension = os.path.basename(full_path)\n",
        "    file_name_without_extension, _ = os.path.splitext(file_name_with_extension)\n",
        "    return file_name_without_extension"
      ],
      "metadata": {
        "id": "SzJLQCHmIjH-"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_background_music(input_file_path, output_dir_path, log_idx):\n",
        "\n",
        "    input_file_name = input_file_path.split('/')[-1].split('_mono.wav')[0]\n",
        "    output_file_path = os.path.join(output_dir_path, input_file_name + '.wav')\n",
        "\n",
        "    # Check if the output file already exists\n",
        "    if os.path.exists(output_file_path):\n",
        "        print(f\"Skipping file {input_file_name}. Vocals file exists.\")\n",
        "        return output_file_path\n",
        "\n",
        "    # Define duration of each chunk in milliseconds (10 minutes)\n",
        "    chunk_duration = 10 * 60 * 1000\n",
        "\n",
        "    # Load input audio\n",
        "    audio = AudioSegment.from_file(input_file_path)\n",
        "\n",
        "    # Get total duration of input audio in milliseconds\n",
        "    total_duration = len(audio)\n",
        "\n",
        "    # Initialize empty list to store processed audio chunks\n",
        "    processed_chunks = []\n",
        "\n",
        "    # Split the audio into chunks of 10 minutes, process each chunk, and store processed chunks\n",
        "    for i in range(0, total_duration, chunk_duration):\n",
        "        start_time = i\n",
        "        end_time = min(i + chunk_duration, total_duration)\n",
        "\n",
        "        # Extract the chunk\n",
        "        input_file_name = get_file_name(input_file_path)\n",
        "        basename = f'{input_file_name}_temp_chunk_{log_idx}.wav'\n",
        "\n",
        "        chunk = audio[start_time:end_time]\n",
        "        temp_input_file_path = os.path.join('spleeter', basename)\n",
        "        chunk.export(temp_input_file_path, format='wav')\n",
        "\n",
        "        # Process the chunk to remove background music\n",
        "        temp_output_folder_name = f'temp_output_{log_idx}'\n",
        "\n",
        "        while True:     # Used to handle Cuda out of memory in which spleeter doesn't create output file\n",
        "            subprocess.run(['python3', '-m', 'spleeter', 'separate', basename, '-o', temp_output_folder_name, '-c', 'wav', '-b', '128k'], cwd='spleeter')\n",
        "\n",
        "            try:\n",
        "                # Load processed chunk\n",
        "                processed_chunk = AudioSegment.from_file(f'spleeter/{temp_output_folder_name}/{basename.split(\".\")[0]}/vocals.wav')\n",
        "                break\n",
        "            except FileNotFoundError:\n",
        "                pass\n",
        "\n",
        "\n",
        "\n",
        "        # Add processed chunk to the list\n",
        "        processed_chunks.append(processed_chunk)\n",
        "\n",
        "    # Concatenate processed chunks\n",
        "    concatenated_audio = processed_chunks[0]\n",
        "    for processed_chunk in processed_chunks[1:]:\n",
        "        concatenated_audio += processed_chunk\n",
        "\n",
        "    # Export concatenated audio to output file\n",
        "\n",
        "    concatenated_audio.export(output_file_path, format='wav')\n",
        "\n",
        "    os.remove(input_file_path)\n",
        "    os.remove(temp_input_file_path)\n",
        "    shutil.rmtree(f'spleeter/{temp_output_folder_name}/{basename.split(\".\")[0]}/')\n",
        "\n",
        "    return output_file_path"
      ],
      "metadata": {
        "id": "VGRTyDUAIgbr"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Remove Silent Moments\n",
        "It is desirable to have audio files that do not have long periods of silence. So we remove the silent parts of the audio longer than a second using the [pydub](https://github.com/jiaaro/pydub) library as a post-processing step."
      ],
      "metadata": {
        "id": "373Pzb_MJqrs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "from pydub import AudioSegment\n",
        "from pydub.silence import split_on_silence"
      ],
      "metadata": {
        "id": "4UnxniBsOg31"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_silent_parts(input_file_path, output_dir_path, metadata, min_silence_len=1000, silence_thresh=-50, keep_silence=1000):\n",
        "    input_file_name = input_file_path.split('/')[-1]\n",
        "    output_file_path = os.path.join(output_dir_path, input_file_name)\n",
        "\n",
        "    # Check if the output file already exists\n",
        "    if os.path.exists(output_file_path):\n",
        "        print(f\"Skipping file {input_file_name}. Silence removed file exists.\")\n",
        "        return output_file_path\n",
        "\n",
        "    # Split audio to silent parts by duration more than `min_silence_len` and keep only `keep_silence` of silence\n",
        "    audio = AudioSegment.from_file(input_file_path, format='wav')\n",
        "    parts = split_on_silence(audio, min_silence_len=min_silence_len, silence_thresh=silence_thresh, keep_silence=keep_silence)\n",
        "\n",
        "    # Concat the silence removed parts\n",
        "    output = AudioSegment.empty()\n",
        "    for part in parts:\n",
        "        output += part\n",
        "\n",
        "    # Write the silence removed audio file\n",
        "    output.export(output_file_path, format='wav')\n",
        "\n",
        "    # Update the duration in metadata\n",
        "    df = pd.read_csv(metadata)\n",
        "    row_index = df.index[df['Audio'] == input_file_name].tolist()\n",
        "    df.at[row_index[0], 'Silence Removed Duration'] = output.duration_seconds\n",
        "    df.to_csv(metadata, index=False)\n",
        "\n",
        "    return output_file_path"
      ],
      "metadata": {
        "id": "LD2xq78yKBDc"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Transcription Module\n",
        "This module uses several Persian ASR models to get a list of reliable transcripts. According to your computational resources including RAM, you can comment out some of the ASR models. We have commented Vosk and Whisper for this notebook.\n",
        "\n",
        "As explained in the original paper (link to be updated), the ASR models have been evaluated and sorted in the order of their reliability. To checkout the evaluation of these models, please refer to this repository (link to be updated)."
      ],
      "metadata": {
        "id": "mAtMVzOLOAXG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "import numpy as np\n",
        "import librosa\n",
        "import torch\n",
        "import torchaudio\n",
        "import wave\n",
        "import json\n",
        "# import pyaudioconvert as pac\n",
        "import uuid\n",
        "import csv\n",
        "import pandas as pd\n",
        "from pydub import AudioSegment\n",
        "from hazm import word_tokenize\n",
        "from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor\n",
        "from hezar.models import Model as HezarModel\n",
        "# from vosk import Model as VoskModel\n",
        "# from vosk import KaldiRecognizer, SetLogLevel\n",
        "# from speechbrain.inference.ASR import WhisperASR"
      ],
      "metadata": {
        "id": "SvFROnr6Omph"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cuda = \"cuda:0\"\n",
        "wav2vec_v3_model_name = \"m3hrdadfi/wav2vec2-large-xlsr-persian-v3\"\n",
        "device = torch.device(cuda if torch.cuda.is_available() else \"cpu\")\n",
        "wav2vec_v3_processor = Wav2Vec2Processor.from_pretrained(wav2vec_v3_model_name)\n",
        "wav2vec_v3_model = Wav2Vec2ForCTC.from_pretrained(wav2vec_v3_model_name).to(device)\n",
        "\n",
        "wav2vec_fa_model_name = \"masoudmzb/wav2vec2-xlsr-multilingual-53-fa\"\n",
        "wav2vec_fa_processor = Wav2Vec2Processor.from_pretrained(wav2vec_fa_model_name)\n",
        "wav2vec_fa_model = Wav2Vec2ForCTC.from_pretrained(wav2vec_fa_model_name).to(device)\n",
        "\n",
        "hezar_model = HezarModel.load(\"hezarai/whisper-small-fa\").to(device)\n",
        "\n",
        "# # You can set log level to -1 to disable debug messages\n",
        "# SetLogLevel(0)\n",
        "# vosk_model = VoskModel(model_name=\"vosk-model-fa-0.5\")\n",
        "\n",
        "# whisper_model = WhisperASR.from_hparams(source=\"speechbrain/asr-whisper-large-v2-commonvoice-fa\", run_opts={\"device\":cuda}).to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 603,
          "referenced_widgets": [
            "7d87759ae07c4210b105536ed8f1d7b8",
            "6967b2ba72414d348705e9a2163021f9",
            "8e8a551ff24547bfacb8eab28d35b0d8",
            "1fdc233270f94eab917aef9bc11d9d8f",
            "ddc638b85c2049508c96af8799474a47",
            "953effc86fa64c3d8fdf0ea11bf3a044",
            "b597cb1773964e7db459776ceac2c735",
            "3e22b35759544791a5b5cbf40099c579",
            "4bece1fac8f0403699647636a6ef49fc",
            "ab07a1dac600426eb90d83bb57bab070",
            "77e0d132932044f68dc896d5a6162367",
            "9d4decba24d042da940773f87336daa5",
            "127849756d73496aa485054aaf509fc9",
            "aae59bd3434b44c68319798cca4eeb98",
            "fd181c6b96ec46c2b3f653de1c450d5e",
            "9675951567eb4a199b7c3910ec73ca1a",
            "37a7b341e91640729046040b11da4d78",
            "20c3ad6ff91642cf91f62ac4e13bc355",
            "c6695c4370d544dea3cba08b0ad26e9f",
            "9dc9ebdaddb947b188aecff65ade9b0c",
            "bbe21b0610124b4296db413832b1a465",
            "df79fab4ce714f969be2dc45cdfa606e",
            "33033c8887e6496b886376004b3abfdc",
            "b6edfeaaf5c9494c9b2140c984948b96",
            "6d6faa657d3a4b32ae2e8df6e20a7b99",
            "8301ceb0f67f45abb2207f124e7c3beb",
            "2a5c05f51cd841908f686bd7c156e617",
            "78bcd54be525451aa8964f9ea6208579",
            "22f7a43f2e1843e4baf235c25cfd597f",
            "5296356123b74b93a5ef94610e0cbe40",
            "0ef0b9d6329441b6ad18934e0bb0c5e3",
            "d525af4089aa4dfe89f18dfc956cb4a5",
            "56037e23fa384dac9bfde200eb1db9a9",
            "f4aa391be68847d9ae1e973b9b874d2d",
            "97311be152ae4d628ba9d06c9b68cabd",
            "4a1ea6ff4f1249f69840c59b4fcf4a5a",
            "2e02094c31aa47d4905e1129cbd27624",
            "063a7232736f4d369614f31e848f7531",
            "613290fa65b646d9ba46eb4d55555045",
            "c53b18a97d2b468788ba926cf7ac6fd4",
            "6702d401cb944e5aafdcdc6f7769dacb",
            "01a64c76277a443abed60cd534da6556",
            "c47233985c014504b3d317adb0b43dff",
            "246b1b72531f45608a5d1457e4848468",
            "f27b8be3b1d4472d96a21971335b7b2f",
            "365a95b6c49e41139f56283d3ab99d63",
            "e85e8aaa8f6c487896d3e9a54adee1fb",
            "bed074073ff04bc084c2001d54f7a57c",
            "693e5102e8cc4f4ca17617c5cd005d6e",
            "ad03933e04ac4b45b039e8c1e6fc00da",
            "3377744cdcf44b8f868c4431008c69c1",
            "09f4eb7e45e343a6a1d61df8fdfc40a9",
            "b7b898050fa14359a7c22882d7aa2e4b",
            "6b48a22604e34601b3dbb39f83baabcb",
            "5c993dc224864323875e215e92abb3d3",
            "0611b5b810d143dda03ac9d091a38817",
            "26b9c548bd0e44a295eaca7d3c55cc8c",
            "71f09ed0ce7f4c12b0939ae8c7c480df",
            "79ee671362ce43b9aebe2c99e6f28b33",
            "64c70b973cb44e4ea9d5f16988b7922c",
            "b54fbf79be8147b391e3d5748316a076",
            "46f94d59206245fb9f11f672286070f0",
            "71a0d24e426a4be4865820e59a9145b0",
            "ef668c8e047948f6a993f4a1b36355ca",
            "b04fe8c4a4854135a7b60a323849a489",
            "c062bb6d60764337abf49170cd9105cc",
            "e22365646a27465588ed9db99b3c6369",
            "44ffeb102ac248a79ed8a78e88e720a6",
            "abdf2df3309b43d2b269b58e4e76ab35",
            "63bff76331a54effb3f0e55b5bcc0a1c",
            "c216bdeadf2a47d08ec45aa307b49407",
            "d7ac9f9ed7a04cb48c170d5d8480f948",
            "c3d748cbbf8a4074a91e48091053ee0a",
            "9f494ce2d4d0488eb7b990b85792a102",
            "4fe2a439cbea4c768ab08b0ebba50084",
            "77ab59b3486f479783b0a28d018f09b8",
            "8092bf7be17b4a96a3965b8326b2916c",
            "605bef355a4045138dcc4dd8cd7c0e7f",
            "e5ec0cbeecac4d9daae25bd3bda378b3",
            "22886044c08642bcb82ce8a0903220ca",
            "f57d3a435a544bb1b6964bf4b92ee608",
            "a99b0591024d406ebe73c6dcab03b9c4",
            "1cf7c9639b59465fa9d46e0a786a952c",
            "593efdaeb4ff403c813ed00a41c28674",
            "720944f73de24edf9c7a59a00351f4a9",
            "7cd7fe1df9734190b376d7297cabaeba",
            "5da4239df1264534b0f91b152dabd8a9",
            "a5f057d15f8d48d985ae3e43c0a0ec0c",
            "e643830ab2f849b29f3e825dbea4b209",
            "ad5687828ed049979d7cb284476a1367",
            "8adbe31e6c5b4bbe9799dc2578226b72",
            "fd1efb1aa2394cdeab00764a294671c2",
            "8c679574dcf9473087419d53ea203d88",
            "690d6e20d2cd484d8235ef497e626ea6",
            "f884ea959bb9417e837da1b3f83df8b3",
            "0a9ee970b8ac411a9ce7c0bbb48519c5",
            "ee7b0784f1664115aa9d545e69c56e35",
            "8b52ec593133441ea3a0e97c669b1040",
            "50c44d9257324d8e992e4c1430c9b244",
            "f18380e717264bd3b48a0c0fffc000a2",
            "383584857b5246359020c0073f747312",
            "ceaadfce5ab145bbafdba74c53519d90",
            "87b6cb5f2c734851b61fe7c85e379faa",
            "7b66aa1aecc64501b5e4007661ed49af",
            "548264c266814a5abf0d12d5915b713a",
            "e9de7a80364a4cecb158c1a3575fc5ff",
            "4986183c8e09464cb5ef650633041763",
            "c00d0942c6c64158bb41820bc9ccb696",
            "bea2ea0bee044dc58c45f88461bf82d5",
            "0327b88f29804fa380aebdf1729fe01d",
            "7eb13221c89545f0a313033040257409",
            "dcf4a328117c435990c19a55824fdc64",
            "fd1aa8c3660547119262d3caefec0d56",
            "f4ef179c9df847b989e9e30fef7e078a",
            "ff5b54e5cbe8445b95759d984e4166d0",
            "62da430f2248422ebe8afa91df674c7c",
            "dcc7cc933413442780e1567f2daeb564",
            "5df02567bc59454585b34a19d85858c2",
            "38f12986180e457dac8f91d4d6d1873c",
            "b5bd7cb633484e3f9286075f98690576",
            "f3fe8befcdcb4848a3f62904c194e7d5",
            "14a70290de474da3a381559a91854887",
            "113a6f4f78fd47c1b3eda612089b6e95",
            "2b2dc1c9828d4e70b5fbd5e9d3be2c14",
            "dc3d153b0197411fa9cad9b60b76861f",
            "e81e60e5ea544117acdc3bb1b7f21ad9",
            "01b3a7c1f32a43478e1d665c60192ef8",
            "d1d72169688841eebf478519464dd321",
            "2c168f5641824b7f9b7e7d88248c82f5",
            "8337feb0025c4c06b7be9d63d468a19a",
            "674ef00642e1447387fbe2aa93d12148",
            "dc877490474446dda6b8646638e38055"
          ]
        },
        "id": "Pim20zlPPQEH",
        "outputId": "4394d9ec-faff-4034-8e76-849e2820be79"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/1.26G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7d87759ae07c4210b105536ed8f1d7b8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at m3hrdadfi/wav2vec2-large-xlsr-persian-v3 were not used when initializing Wav2Vec2ForCTC: ['wav2vec2.encoder.pos_conv_embed.conv.weight_g', 'wav2vec2.encoder.pos_conv_embed.conv.weight_v']\n",
            "- This IS expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at m3hrdadfi/wav2vec2-large-xlsr-persian-v3 and are newly initialized: ['wav2vec2.encoder.pos_conv_embed.conv.parametrizations.weight.original0', 'wav2vec2.encoder.pos_conv_embed.conv.parametrizations.weight.original1']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "preprocessor_config.json:   0%|          | 0.00/214 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9d4decba24d042da940773f87336daa5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/307 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "33033c8887e6496b886376004b3abfdc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.57k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f4aa391be68847d9ae1e973b9b874d2d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/399 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f27b8be3b1d4472d96a21971335b7b2f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/85.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0611b5b810d143dda03ac9d091a38817"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/1.26G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e22365646a27465588ed9db99b3c6369"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at masoudmzb/wav2vec2-xlsr-multilingual-53-fa were not used when initializing Wav2Vec2ForCTC: ['wav2vec2.encoder.pos_conv_embed.conv.weight_g', 'wav2vec2.encoder.pos_conv_embed.conv.weight_v']\n",
            "- This IS expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at masoudmzb/wav2vec2-xlsr-multilingual-53-fa and are newly initialized: ['wav2vec2.encoder.pos_conv_embed.conv.parametrizations.weight.original0', 'wav2vec2.encoder.pos_conv_embed.conv.parametrizations.weight.original1']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_config.yaml:   0%|          | 0.00/953 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "605bef355a4045138dcc4dd8cd7c0e7f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.pt:   0%|          | 0.00/967M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e643830ab2f849b29f3e825dbea4b209"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "(…)eprocessor/feature_extractor_config.yaml:   0%|          | 0.00/225 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f18380e717264bd3b48a0c0fffc000a2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "preprocessor/tokenizer_config.yaml:   0%|          | 0.00/1.84k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7eb13221c89545f0a313033040257409"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "preprocessor/tokenizer.json:   0%|          | 0.00/2.20M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "14a70290de474da3a381559a91854887"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Regular expression pattern to match words containing at least one word character\n",
        "word_pattern = re.compile(r'\\w+')\n",
        "\n",
        "def get_word_count(text):\n",
        "    words = word_tokenize(text)\n",
        "    valid_words = [word for word in words if word_pattern.match(word)]\n",
        "    return len(valid_words)"
      ],
      "metadata": {
        "id": "1td6aAjzPV21"
      },
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def wav2vec_v3_speech_file_to_array_fn(path):\n",
        "    speech_array, sampling_rate = torchaudio.load(path)\n",
        "    speech_array = speech_array.squeeze().numpy()\n",
        "    speech_array = librosa.resample(np.asarray(speech_array), orig_sr=sampling_rate, target_sr=wav2vec_v3_processor.feature_extractor.sampling_rate)\n",
        "\n",
        "    return speech_array"
      ],
      "metadata": {
        "id": "kYlyazPgPYYm"
      },
      "execution_count": 98,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def wav2vec_v3_transcript(audio_file_path):\n",
        "    speech = wav2vec_v3_speech_file_to_array_fn(audio_file_path)\n",
        "\n",
        "    features = wav2vec_v3_processor(\n",
        "        speech,\n",
        "        sampling_rate=wav2vec_v3_processor.feature_extractor.sampling_rate,\n",
        "        return_tensors=\"pt\",\n",
        "        padding=True\n",
        "    )\n",
        "\n",
        "    input_values = features.input_values.to(device)\n",
        "    attention_mask = features.attention_mask.to(device)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        logits = wav2vec_v3_model(input_values, attention_mask=attention_mask).logits\n",
        "\n",
        "    pred_ids = torch.argmax(logits, dim=-1)\n",
        "\n",
        "    predicted = wav2vec_v3_processor.batch_decode(pred_ids)\n",
        "    return predicted[0]"
      ],
      "metadata": {
        "id": "eHXj5gmrPcBr"
      },
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def wav2vec_fa_speech_file_to_array_fn(path):\n",
        "    speech_array, sampling_rate = torchaudio.load(path)\n",
        "    speech_array = speech_array.squeeze().numpy()\n",
        "    speech_array = librosa.resample(np.asarray(speech_array), orig_sr=sampling_rate, target_sr=wav2vec_fa_processor.feature_extractor.sampling_rate)\n",
        "\n",
        "    return speech_array"
      ],
      "metadata": {
        "id": "uiaRNuZCP6Fg"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def wav2vec_fa_transcript(audio_file_path):\n",
        "    speech = wav2vec_fa_speech_file_to_array_fn(audio_file_path)\n",
        "\n",
        "    features = wav2vec_fa_processor(\n",
        "        speech,\n",
        "        sampling_rate=wav2vec_fa_processor.feature_extractor.sampling_rate,\n",
        "        return_tensors=\"pt\",\n",
        "        padding=True\n",
        "    )\n",
        "\n",
        "    input_values = features.input_values.to(device)\n",
        "    attention_mask = features.attention_mask.to(device)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        logits = wav2vec_fa_model(input_values, attention_mask=attention_mask).logits\n",
        "\n",
        "    pred_ids = torch.argmax(logits, dim=-1)\n",
        "\n",
        "    predicted = wav2vec_fa_processor.batch_decode(pred_ids)\n",
        "    return predicted[0]"
      ],
      "metadata": {
        "id": "TRCdWED_QABd"
      },
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def hezar_transcript(audio_file_path):\n",
        "    transcript = hezar_model.predict(audio_file_path)\n",
        "    transcript = transcript[0]['text']\n",
        "\n",
        "    return transcript.strip()"
      ],
      "metadata": {
        "id": "61znOdD2QJwf"
      },
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# def vosk_transcript(audio_file_path):\n",
        "#     # Generate a unique identifier for the file names\n",
        "#     unique_id = uuid.uuid4()\n",
        "\n",
        "#     # Create an output file name with a random suffix\n",
        "#     temp_output_file_name = f\"{unique_id}.{audio_file_path.split('.')[-1]}\"\n",
        "\n",
        "#     # Load the audio file\n",
        "#     audio = AudioSegment.from_mp3(audio_file_path)\n",
        "#     # Export the audio in WAV format\n",
        "#     audio.export(temp_output_file_name, format=\"wav\")\n",
        "\n",
        "#     # Convert the WAV file to 16-bit\n",
        "#     pac.convert_wav_to_16bit_mono(temp_output_file_name, temp_output_file_name)\n",
        "\n",
        "#     wf = wave.open(temp_output_file_name, \"rb\")\n",
        "\n",
        "#     if wf.getnchannels() != 1 or wf.getsampwidth() != 2 or wf.getcomptype() != \"NONE\":\n",
        "#         print(\"Audio file must be WAV format mono PCM.\")\n",
        "#         return ''\n",
        "\n",
        "#     rec = KaldiRecognizer(vosk_model, wf.getframerate())\n",
        "#     rec.SetWords(True)\n",
        "#     rec.SetPartialWords(True)\n",
        "\n",
        "#     while True:\n",
        "#         data = wf.readframes(4000)\n",
        "#         if len(data) == 0:\n",
        "#             break\n",
        "#         if rec.AcceptWaveform(data):\n",
        "#             rec.Result()\n",
        "#         else:\n",
        "#             rec.PartialResult()\n",
        "\n",
        "#     os.remove(temp_output_file_name)\n",
        "#     return json.loads(rec.FinalResult())['text']"
      ],
      "metadata": {
        "id": "wZTFVsUCQM4r"
      },
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# def whisper_transcript(audio_file_path):\n",
        "#     transcript = whisper_model.transcribe_file(audio_file_path)\n",
        "\n",
        "#     symbolic_link_file_path = audio_file_path.split('/')[-1]\n",
        "#     if os.path.islink(symbolic_link_file_path):\n",
        "#         os.remove(symbolic_link_file_path)\n",
        "\n",
        "#     return transcript"
      ],
      "metadata": {
        "id": "cXD9fGstQOC1"
      },
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# With zero having lest avg cer and 3 most\n",
        "# asrs_dict = {'Vosk': 0, 'Wav2Vec-V3': 1, 'Wav2Vec-FA': 2, 'Whisper': 3, 'Hezar': 4}\n",
        "asrs_dict = {'Wav2Vec-V3': 1, 'Wav2Vec-FA': 2, 'Hezar': 3}\n",
        "\n",
        "# asr_transcripts = [('Hezar', hezar_transcript), ('Wav2Vec-V3', wav2vec_v3_transcript), ('Whisper', whisper_transcript), ('Vosk', vosk_transcript), ('Wav2Vec-FA', wav2vec_fa_transcript)]\n",
        "asr_transcripts = [('Hezar', hezar_transcript), ('Wav2Vec-V3', wav2vec_v3_transcript), ('Wav2Vec-FA', wav2vec_fa_transcript)]"
      ],
      "metadata": {
        "id": "Y1NqFfbBQYHV"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Used to store the transcript of audio chunks\n",
        "transcripts_metadata_path = 'transcripts_metadata.csv'\n",
        "\n",
        "# Create the metadata file with header\n",
        "if not os.path.exists(transcripts_metadata_path):\n",
        "    with open(transcripts_metadata_path, 'w', newline='') as csvfile:\n",
        "        writer = csv.writer(csvfile)\n",
        "        writer.writerow(['Audio', *[asr_name for asr_name, _ in asr_transcripts]])\n",
        "\n",
        "transcripts_metadata = pd.read_csv(transcripts_metadata_path)"
      ],
      "metadata": {
        "id": "diuJVhfEQaaI"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_best_transcripts(audio_file_path, output_chunk_path='', log=False):\n",
        "    transcripts = []\n",
        "\n",
        "    # Get transcript of all ASRs\n",
        "    for asr, asr_transcript in asr_transcripts:\n",
        "        transcript = asr_transcript(audio_file_path)\n",
        "        transcripts.append((asr, transcript, get_word_count(transcript)))\n",
        "\n",
        "    # If log is True, save the transcripts to the transcripts_metadata csv file defined above\n",
        "    if log == True:\n",
        "        with open(transcripts_metadata_path, 'a', newline='') as csvfile:\n",
        "            writer = csv.writer(csvfile)\n",
        "            writer.writerow([output_chunk_path, *[transcript for _, transcript, _ in transcripts]])\n",
        "\n",
        "    # Filter out corrupt transcripts like \"من من من من من من من\" with repetitive patterns\n",
        "    transcripts_filter_for_repetition = [(asr, transcript, word_count) for (asr, transcript, word_count) in transcripts if not re.search(r'(.{1,5})(\\1{5,})|(.{5,30})(\\3{3,})|(.{30,})(\\5{2,})', transcript)]\n",
        "\n",
        "    # Sort transcripts based on length and get the max length\n",
        "    transcripts_sorted_by_word_count = sorted(transcripts_filter_for_repetition, key=lambda x: x[2], reverse=True)\n",
        "    max_word_count = transcripts_sorted_by_word_count[0][2]\n",
        "\n",
        "    # Filter out transcripts that are shorter than 80% of the max length of transcripts\n",
        "    transcripts_filtered_for_truncated = [(asr, transcript) for (asr, transcript, word_count) in transcripts_sorted_by_word_count if word_count >= 0.8 * max_word_count]\n",
        "\n",
        "    # Sort transcripts based on ASR reliablity order defined above\n",
        "    transcripts_sorted_by_least_avg_cer = sorted(transcripts_filtered_for_truncated, key=lambda x: asrs_dict[x[0]])\n",
        "\n",
        "    return transcripts_sorted_by_least_avg_cer"
      ],
      "metadata": {
        "id": "cFgjE4TAOCnb"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Start-End Alignment\n",
        "There might be some mismatches at the start and end of the audio and text files. As an example, the speaker might read the title and author name while it is not in the text, or the text might have some additional sentences at the end which are not read by the speaker. This section attempts to match the audio and text files in the start and end by removing a few seconds/words from each."
      ],
      "metadata": {
        "id": "t-PAXfLpKKwV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import os\n",
        "from pydub import AudioSegment\n",
        "from hazm import Normalizer\n",
        "from jiwer import cer\n",
        "from pydub import AudioSegment\n",
        "from pydub.silence import split_on_silence"
      ],
      "metadata": {
        "id": "cgv3LzHTOsuD"
      },
      "execution_count": 185,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "non_word_char_pattern = r\"[\\s«»()!*[\\]\\\"'^\\-+=_–—&<>;٫,.?:؛/{}،؟]\"\n",
        "\n",
        "def match_special_character(char):\n",
        "    return bool(re.match(non_word_char_pattern, char))"
      ],
      "metadata": {
        "id": "QBcMuFL3Khkc"
      },
      "execution_count": 186,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_non_word_chars(text):\n",
        "    pattern = r'[^\\w\\s]'\n",
        "    cleaned_text = re.sub(pattern, ' ', text)\n",
        "    return cleaned_text"
      ],
      "metadata": {
        "id": "LdSniWBpKlrc"
      },
      "execution_count": 187,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_white_spaces(text):\n",
        "    cleaned_text = re.sub(r'\\s+', ' ', text)\n",
        "    return cleaned_text.strip()"
      ],
      "metadata": {
        "id": "mlNeA9RNKmtY"
      },
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_word_only_text(text):\n",
        "  word_only_text = remove_non_word_chars(text)\n",
        "  extra_space_removed_text = remove_white_spaces(word_only_text)\n",
        "\n",
        "  return extra_space_removed_text"
      ],
      "metadata": {
        "id": "EXJhiVYOKohQ"
      },
      "execution_count": 189,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_texts_cer_with_lookup(reference_text, transcript, lookup, direction='start'):\n",
        "  \"\"\"\n",
        "  Returns CER of either the starting or ending #lookup characters of the two texts\n",
        "  \"\"\"\n",
        "\n",
        "  # Preprocess input texts to only contain word characters\n",
        "  word_only_reference_text = get_word_only_text(reference_text)\n",
        "  word_only_transcript = get_word_only_text(transcript)\n",
        "\n",
        "  # Return +infinity for CER if any of the texts is empty\n",
        "  if not word_only_reference_text.strip() or not word_only_transcript.strip():\n",
        "    return float('inf')\n",
        "\n",
        "  # Get CER of #lookup start characters of the texts\n",
        "  if direction == 'start':\n",
        "    return cer(word_only_reference_text[:lookup], word_only_transcript[:lookup])\n",
        "\n",
        "  # Get CER of #lookup last characters of the texts\n",
        "  else:\n",
        "    return cer(word_only_reference_text[-lookup:], word_only_transcript[-lookup:])"
      ],
      "metadata": {
        "id": "4fqAJnpPKqpa"
      },
      "execution_count": 190,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def match_texts_from_start(reference_text, transcript, max_index=500, min_lookup=50, max_lookup=150):\n",
        "    \"\"\"\n",
        "    Tries to find the best match to the transcript from the start of the reference text\n",
        "\n",
        "    Arguments:\n",
        "    - reference_text: ground truth text from which the true transcript is extracted\n",
        "    - transcript: the hypothesis transcript from ASR\n",
        "    - max_index: the maximum index to search for starting cut index\n",
        "    - min_lookup: in case the texts are short, the max_index would only go as far as min_lookup characters are there for comparison\n",
        "    - max_lookup: determines the number of characters that is compared in the selected reference text and hypothesis\n",
        "\n",
        "    Returns:\n",
        "    - CER of the parts matched\n",
        "    - the index for which reference_text[i:] is the best match for the transcript\n",
        "    \"\"\"\n",
        "\n",
        "    min_cer = float('inf')\n",
        "    best_index = 0\n",
        "\n",
        "    for i in range(min(len(reference_text) - min_lookup, max_index)):\n",
        "        # Ensures the reference text is split in word boundaries only\n",
        "        if i != 0 and not (match_special_character(reference_text[i - 1]) and not match_special_character(reference_text[i])): continue\n",
        "\n",
        "        # Calculate maximum number of characters that are available to compare the selected text and hypothesis\n",
        "        min_left = min(len(transcript), len(reference_text) - (i + 1))\n",
        "        lookup = min(min_left, max_lookup)\n",
        "\n",
        "        cer_value = get_texts_cer_with_lookup(reference_text[i:], transcript, lookup)\n",
        "\n",
        "        # Store the best search result\n",
        "        if cer_value < min_cer:\n",
        "            min_cer = cer_value\n",
        "            best_index = i\n",
        "\n",
        "    return min_cer, best_index"
      ],
      "metadata": {
        "id": "HcWcJh0dKvub"
      },
      "execution_count": 191,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fix transcript and tune the end index of the reference text for least cer\n",
        "def match_texts_from_end(reference_text, transcript, max_index=600, min_lookup=50, max_lookup=150):\n",
        "    \"\"\"\n",
        "    Tries to find the best match to the transcript from the end of the reference text\n",
        "\n",
        "    Arguments:\n",
        "    - reference_text: ground truth text from which the true transcript is extracted\n",
        "    - transcript: the hypothesis transcript from ASR\n",
        "    - max_index: the maximum index to search for ending cut index\n",
        "    - min_lookup: in case the texts are short, the max_index would only go as far as min_lookup characters are there for comparison\n",
        "    - max_lookup: determines the number of characters that is compared in the selected reference text and hypothesis\n",
        "\n",
        "    Returns:\n",
        "    - CER of the parts matched\n",
        "    - the index for which reference_text[:i] is the best match for the transcript\n",
        "    \"\"\"\n",
        "\n",
        "    min_cer = float('inf')\n",
        "    best_index = 0\n",
        "\n",
        "    for i in range(len(reference_text) - 1, max(len(reference_text) - min_lookup, max_index) - 1, -1):\n",
        "        # Calculate maximum number of characters that are available to compare the selected text and hypothesis\n",
        "        min_left = min(len(transcript), i + 1)\n",
        "        lookup = min(min_left, max_lookup)\n",
        "\n",
        "        # Ensures the reference text is split in word boundaries only\n",
        "        if i != len(reference_text) - 1 and not (match_special_character(reference_text[i]) and not match_special_character(reference_text[i + 1])): continue\n",
        "\n",
        "        cer_value = get_texts_cer_with_lookup(reference_text[:i], transcript, lookup, direction='end')\n",
        "\n",
        "        # Store the best search result\n",
        "        if cer_value < min_cer:\n",
        "            min_cer = cer_value\n",
        "            best_index = i\n",
        "\n",
        "    return min_cer, best_index"
      ],
      "metadata": {
        "id": "096x4EZXKzjW"
      },
      "execution_count": 192,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def match_with_transcript(reference_text, transcript, match_function):\n",
        "    \"\"\"\"\n",
        "    Searches the index of the reference text from which it is a good match to the transcript\n",
        "\n",
        "    Arguments:\n",
        "    - reference_text: ground truth text from which the true transcript is extracted\n",
        "    - transcript: the hypothesis transcript from ASR\n",
        "    - match_function: either match_texts_from_start or match_texts_from_end\n",
        "\n",
        "    Returns:\n",
        "    - CER of the best match found\n",
        "    - index for which reference_text[i:] or reference_text[:i] is the best match to the transcript\n",
        "    \"\"\"\n",
        "\n",
        "    processed_transcript = normalize_text(transcript)\n",
        "    cer_value, index = match_function(reference_text, processed_transcript)\n",
        "    return cer_value, index"
      ],
      "metadata": {
        "id": "jL74kyFlK2Fd"
      },
      "execution_count": 193,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def fill_parts_transcript(parts, transcripts, start_index, end_index, log_idx):\n",
        "  \"\"\"\"\n",
        "    Inputs a list of audio chunks with a corresponding list of extracted transcripts\n",
        "    and fills in missing transcripts in the range transcripts[start_index:end_index]\n",
        "  \"\"\"\n",
        "\n",
        "  out_file = f\"./temp_constructed_audio_{log_idx}.wav\"\n",
        "\n",
        "\n",
        "  for i in range(start_index, end_index):\n",
        "    if transcripts[i] == None:\n",
        "      parts[i].export(out_file, format=\"wav\")\n",
        "      audio_transcript = get_best_transcripts(out_file)[0][1]\n",
        "      transcripts[i] = audio_transcript\n",
        "\n",
        "  if os.path.exists(out_file): os.remove(out_file)"
      ],
      "metadata": {
        "id": "kQ95gvYuK4gx"
      },
      "execution_count": 194,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_parts_by_max_duration(parts, max_duration, direction='start'):\n",
        "    \"\"\"\n",
        "    Inputs a list of audio chunks, selects as many chunks from start(end) of the chunks as they sum up to more than `max_duration`\n",
        "    \"\"\"\n",
        "\n",
        "    # Determines if the parts are going to be selected from start or end of list\n",
        "    slice_step = 1 if direction == 'start' else -1\n",
        "\n",
        "    selected_parts = []\n",
        "    current_diration_seconds = 0\n",
        "\n",
        "    for part in parts[::slice_step]:\n",
        "      selected_parts.append(part)\n",
        "      current_diration_seconds += part.duration_seconds\n",
        "\n",
        "      # The duration of selected chunks exceeds the max_duration for the first time to break the process\n",
        "      if current_diration_seconds > max_duration:\n",
        "        break\n",
        "\n",
        "    return selected_parts[::slice_step]"
      ],
      "metadata": {
        "id": "5uXEp7R4K6oG"
      },
      "execution_count": 195,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_best_start_match(parts, reference_text, max_lookup_duration, max_segment_duration, cer_threshold=0.2, text_len_diff_threshold=10, log_idx=0):\n",
        "    \"\"\"\n",
        "    Finds the best starting point for an audio regarding how well it matches the reference text\n",
        "\n",
        "    Arguments:\n",
        "    - parts: input audio chunks\n",
        "    - reference_text: the ground truth text for most of the audio except for perhaps its start and end\n",
        "    - max_lookup_duration: maximum time that the function searches in the audio to find best starting point for audio\n",
        "    - max_segment_duration: maximum time that the function matches the audio from an start point with the text\n",
        "    - cer_threshold: acceptable CER value showing a good match between audio and text\n",
        "    - text_len_diff_threshold: indicates the number of characters we may discard in the reference text to find a better CER value.\n",
        "      More specifically, once the cer_threshold is already found in the search, the audio and text are most probaby a good match.\n",
        "      And a better CER value with a smaller subpart of them may only be random due to ASR not being 100 correct. So there is a threshold\n",
        "      for the number of characters we may allow to be skipped for an even better CER value.\n",
        "\n",
        "    Returns:\n",
        "    - CER value of the found match\n",
        "    - text index that gives best match\n",
        "    - audio chunk index that gives best match\n",
        "    \"\"\"\n",
        "\n",
        "    print(\"\\tMatching Start...\")\n",
        "    # Selects as many input chunks for search as max_lookup_duration\n",
        "    starting_parts = get_parts_by_max_duration(parts, max_lookup_duration)\n",
        "    starting_transcripts = [None for _ in range(len(starting_parts))]\n",
        "\n",
        "    # History of found matches\n",
        "    results = []\n",
        "\n",
        "    # If acceptable CER is already found in search\n",
        "    cer_threshold_met = False\n",
        "\n",
        "    # How many further characters do the matching functions suggest to remove from reference text\n",
        "    text_len_diff = 0\n",
        "\n",
        "    # best CER value, starting index of the reference text, and starting chunk of the audio\n",
        "    best_cer, best_text_idx, best_audio_idx = (float('inf'), -1, -1)\n",
        "\n",
        "    for i in range(len(starting_parts)):\n",
        "        # Starting from an input audio chunk, select as many chunks for matching as max_segment_duration\n",
        "        current_parts = get_parts_by_max_duration(starting_parts[i:], max_segment_duration)\n",
        "        constructed_audio = sum(current_parts, AudioSegment.empty())\n",
        "        # temp_constructed_audio_path = 'temp_constructed_audio.wav'\n",
        "        temp_constructed_audio_path = f'temp_constructed_audio_{log_idx}.wav'\n",
        "        constructed_audio.export(temp_constructed_audio_path, format='wav')\n",
        "\n",
        "        # Get transcript for chunks that haven't already been processed\n",
        "        fill_parts_transcript(starting_parts, starting_transcripts, i, i + len(current_parts), log_idx=log_idx)\n",
        "        audio_transcript = ' '.join(starting_transcripts[i:i+len(current_parts)])\n",
        "\n",
        "        # Given the selected chunks and the reference text, find best index of text that reference_text[idx:] is the best match\n",
        "        cer_value, text_index = match_with_transcript(reference_text, audio_transcript, match_texts_from_start)\n",
        "\n",
        "        print(f\"\\t\\tPart [{i}]: Transcript= {audio_transcript}\")\n",
        "        print(f\"\\t\\tMatched Text= {reference_text[text_index:text_index + len(audio_transcript)]}\")\n",
        "        print(f'\\t\\tCER={cer_value}')\n",
        "\n",
        "        # Save to search history\n",
        "        results.append((cer_value, text_index, i))\n",
        "\n",
        "        if len(results) > 1:\n",
        "          text_len_diff = abs(results[-2][1] - results[-1][1])\n",
        "          if cer_threshold_met and (cer_value > results[-1][0] or text_len_diff > text_len_diff_threshold):\n",
        "            # If acceptable CER is already found in search, we wont accept a smaller CER value or a further large jump in text\n",
        "            break\n",
        "\n",
        "        # Set the flag that shows if acceptable CER is found in search resulst\n",
        "        if cer_value < cer_threshold:\n",
        "            cer_threshold_met = True\n",
        "\n",
        "        # Save best search result\n",
        "        best_cer, best_text_idx, best_audio_idx = (cer_value, text_index, i) if cer_value < best_cer else (best_cer, best_text_idx, best_audio_idx)\n",
        "\n",
        "    if os.path.exists(temp_constructed_audio_path): os.remove(temp_constructed_audio_path)\n",
        "    return best_cer, best_text_idx, best_audio_idx"
      ],
      "metadata": {
        "id": "22yb7qO0K_q-"
      },
      "execution_count": 196,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_best_end_match(parts, reference_text, max_lookup_duration, max_segment_duration, cer_threshold=0.2, text_len_diff_threshold=10, log_idx=0):\n",
        "    \"\"\"\n",
        "    Finds the best ending point for an audio regarding how well it matches the reference text\n",
        "\n",
        "    Arguments:\n",
        "    - parts: input audio chunks\n",
        "    - reference_text: the ground truth text for most of the audio except for perhaps its start and end\n",
        "    - max_lookup_duration: maximum time that the function searches in the audio to find best ending point for audio\n",
        "    - max_segment_duration: maximum time that the function matches the audio from an end point with the text\n",
        "    - cer_threshold: acceptable CER value showing a good match between audio and text\n",
        "    - text_len_diff_threshold: indicates the number of characters we may discard in the reference text to find a better CER value.\n",
        "      More specifically, once the cer_threshold is already found in the search, the audio and text are most probaby a good match.\n",
        "      And a better CER value with a smaller subpart of them may only be random due to ASR not being 100 correct. So there is a threshold\n",
        "      for the number of characters we may allow to be skipped for an even better CER value.\n",
        "\n",
        "    Returns:\n",
        "    - CER value of the found match\n",
        "    - text index that gives best match\n",
        "    - audio chunk index that gives best match\n",
        "    \"\"\"\n",
        "\n",
        "    print(\"\\tMatching End...\")\n",
        "    # Selects as many input chunks for search as max_lookup_duration\n",
        "    ending_parts = get_parts_by_max_duration(parts, max_lookup_duration, direction='end')\n",
        "    ending_transcripts = [None for _ in range(len(ending_parts))]\n",
        "\n",
        "    # History of found matches\n",
        "    results = []\n",
        "\n",
        "    # If acceptable CER is already found in search\n",
        "    cer_threshold_met = False\n",
        "\n",
        "    # How many further characters do the matching functions suggest to remove from reference text\n",
        "    text_len_diff = 0\n",
        "\n",
        "    # best CER value, starting index of the reference text, and starting chunk of the audio\n",
        "    best_cer, best_text_idx, best_audio_idx = (float('inf'), -1, -1)\n",
        "\n",
        "    for i in range(len(ending_parts)):\n",
        "        # Starting from an input audio chunk, select as many chunk for matching as max_segment_duration\n",
        "        current_parts = get_parts_by_max_duration(ending_parts[:len(ending_parts) - i], max_segment_duration, direction='end')\n",
        "        constructed_audio = sum(current_parts, AudioSegment.empty())\n",
        "\n",
        "        # constructed_audio.export('temp_constructed_audio.wav', format='wav')\n",
        "        constructed_audio.export(f'temp_constructed_audio_{log_idx}.wav', format='wav')\n",
        "\n",
        "\n",
        "        # Get transcript for chunks that haven't already been processed\n",
        "        fill_parts_transcript(ending_parts, ending_transcripts, len(ending_parts) - i - len(current_parts), len(ending_parts) - i, log_idx=log_idx)\n",
        "\n",
        "        audio_transcript = ' '.join(ending_transcripts[len(ending_parts) - i - len(current_parts):len(ending_parts) - i])\n",
        "        cer_value, text_index = match_with_transcript(reference_text, audio_transcript, match_texts_from_end)\n",
        "\n",
        "        print(f\"\\t\\tPart [{i}]: Transcript= {audio_transcript}\")\n",
        "        print(f\"\\t\\tMatched Text= {reference_text[text_index - len(audio_transcript):text_index]}\")\n",
        "        print(f'\\t\\tCER={cer_value}')\n",
        "\n",
        "        # Save to search history\n",
        "        results.append((cer_value, text_index, i))\n",
        "\n",
        "        if len(results) > 1:\n",
        "          text_len_diff = abs(results[-2][1] - results[-1][1])\n",
        "          if cer_threshold_met and (cer_value > results[-1][0] or text_len_diff > text_len_diff_threshold):\n",
        "            # If acceptable CER is already found in search, we wont accept a smaller CER value or a further large jump in text\n",
        "            break\n",
        "\n",
        "        if cer_value < cer_threshold:\n",
        "            cer_threshold_met = True\n",
        "\n",
        "        # Save best search result\n",
        "        best_cer, best_text_idx, best_audio_idx = (cer_value, text_index, i) if cer_value < best_cer else (best_cer, best_text_idx, best_audio_idx)\n",
        "\n",
        "    return best_cer, best_text_idx, best_audio_idx"
      ],
      "metadata": {
        "id": "tAk2UehILDN0"
      },
      "execution_count": 197,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def split_audio_on_silence(audio_file, silence_threshold=-40, min_silence_len=300):\n",
        "    audio = AudioSegment.from_file(audio_file, format=\"wav\")\n",
        "    parts = split_on_silence(audio, min_silence_len=min_silence_len, silence_thresh=silence_threshold, keep_silence=True)\n",
        "    return parts"
      ],
      "metadata": {
        "id": "2uk5snCSLFua"
      },
      "execution_count": 198,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def start_end_align(audio_file_path, reference_text_path, output_dir_path, max_lookup_duration=60, max_segment_duration=15, cer_threshold=0.2, log_idx=0):\n",
        "    \"\"\"\n",
        "    Finds best starting/ending point for a pair of audio and text files regarding how well they match\n",
        "\n",
        "    Arguments:\n",
        "    - audio_path: path to audio to match with text\n",
        "    - aligned_audio_path: path to output audio, potentially trimmed from start/end\n",
        "    - reference_text_path: path to ground truth text to match with audio\n",
        "    - aligned_text_path: path to output text, potentially trimmed from start/end\n",
        "    - max_lookup_duration: max time duration to search from start/end of the audio for best match\n",
        "    - max_segment_duration: indicates duration of the audio used to compare text and audio as being a complete match for each other\n",
        "    - cer_threshold: acceptable CER value that guarantees a good match\n",
        "    \"\"\"\n",
        "    input_audio_name = audio_file_path.split('/')[-1].split('.')[0]\n",
        "    output_audio_path = os.path.join(output_dir_path, input_audio_name + '.wav')\n",
        "    output_text_path = os.path.join(output_dir_path, input_audio_name + '.txt')\n",
        "\n",
        "    if os.path.exists(output_audio_path):\n",
        "       print(f\"Skipping file {input_audio_name}.wav. Start-End aligned audio exists.\")\n",
        "       return output_audio_path, output_text_path\n",
        "\n",
        "    reference_text = ''\n",
        "\n",
        "    with open(reference_text_path, 'r') as f:\n",
        "      reference_text = f.read()\n",
        "\n",
        "    # Match start of audio with start of reference text\n",
        "    parts = split_audio_on_silence(audio_file_path)\n",
        "\n",
        "    best_match_start = find_best_start_match(parts, reference_text, max_lookup_duration, max_segment_duration, cer_threshold, log_idx=log_idx)\n",
        "\n",
        "    if not best_match_start:\n",
        "      print(\"\\x1b[31m\\\"The audio and text are not a match!\\\"\\x1b[0m\")\n",
        "      raise Exception(\"The audio and text are not a match!\")\n",
        "\n",
        "    # Output start matching result in a temp file to use for end matching\n",
        "    constructed_audio_start = sum(parts[best_match_start[2]:], AudioSegment.empty())\n",
        "    temp_start_aligned_filename = f\"./temp_aligned_start_audio_{log_idx}.wav\"\n",
        "    constructed_audio_start.export(temp_start_aligned_filename, format=\"wav\")\n",
        "\n",
        "    # Update the reference text with the strat match result\n",
        "    reference_text_start = reference_text[best_match_start[1]:]\n",
        "\n",
        "    # Match end of audio with start of reference text\n",
        "    parts = split_audio_on_silence(temp_start_aligned_filename)\n",
        "    best_match_end = find_best_end_match(parts, reference_text_start, max_lookup_duration, max_segment_duration, cer_threshold, log_idx=log_idx)\n",
        "\n",
        "    if not best_match_end:\n",
        "      print(\"\\x1b[31m\\\"The audio and text are not a match!\\\"\\x1b[0m\")\n",
        "      raise Exception(\"The audio and text are not a match!\")\n",
        "\n",
        "    # Export the final start/end aligned audio file\n",
        "    constructed_audio_end = sum(parts[:len(parts) - best_match_end[2]], AudioSegment.empty())\n",
        "    constructed_audio_end.export(output_audio_path, format=\"wav\")\n",
        "\n",
        "    # Export the final start/end aligned text file\n",
        "    aligned_reference_text = reference_text_start[:best_match_end[1]]\n",
        "    with open(output_text_path, 'w') as f:\n",
        "      f.write(aligned_reference_text)\n",
        "\n",
        "    if os.path.exists(temp_start_aligned_filename): os.remove(temp_start_aligned_filename)\n",
        "\n",
        "    return output_audio_path, output_text_path"
      ],
      "metadata": {
        "id": "5Cht9ho8KMwG"
      },
      "execution_count": 199,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Forced Alignemnt\n",
        "In order to prepare the audio and text files for text-to-speech, speech-to-text, ... tasks, we must break them into smaller chunks of a few seconds and a few matching words. This is the forced alignment task. [Aeneas](https://github.com/readbeyond/aeneas) is an awesome tool for this task and you can see an example of Persian forced alignemtn using this task in [this repository](https://github.com/MahtaFetrat/VirgoolInformal-Speech-Dataset). But this tool doesn't work well when the audio and text files are not exact matches as stated in its limitations. So we implemented our own method of forced alignment as follows."
      ],
      "metadata": {
        "id": "lEK8XhmmLlz4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "import time\n",
        "import csv\n",
        "from jiwer import cer\n",
        "from pydub import AudioSegment\n",
        "from pydub.silence import split_on_silence\n",
        "from pydub import AudioSegment\n",
        "from hazm import Normalizer\n",
        "from pydub import AudioSegment\n",
        "from pydub.silence import split_on_silence\n",
        "from hazm import word_tokenize"
      ],
      "metadata": {
        "id": "w6B-aiu_Ouqm"
      },
      "execution_count": 200,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_texts_cer(reference_text, transcript):\n",
        "  \"\"\"\n",
        "  Returns CER of either the two texts\n",
        "  \"\"\"\n",
        "\n",
        "  # Preprocess input texts to only contain word characters\n",
        "  word_only_reference_text = get_word_only_text(reference_text)\n",
        "  word_only_transcript = get_word_only_text(transcript)\n",
        "\n",
        "  # Return +infinity for CER if any of the texts is empty\n",
        "  if not word_only_reference_text.strip() or not word_only_transcript.strip():\n",
        "    return float('inf')\n",
        "\n",
        "  return cer(word_only_reference_text, word_only_transcript)\n"
      ],
      "metadata": {
        "id": "tk8hzkR_eKci"
      },
      "execution_count": 201,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def concat_parts_to_min_duration(parts, min_part_duration=4):\n",
        "  \"\"\"\n",
        "  Merge input audio chunks such that they exceeds a minimum duration and return the resulting merged chunks list\n",
        "  \"\"\"\n",
        "\n",
        "  new_parts = []\n",
        "  current_duration = 0\n",
        "  current_parts = []\n",
        "\n",
        "  for part in parts:\n",
        "    current_parts.append(part)\n",
        "    current_duration += part.duration_seconds\n",
        "\n",
        "    if current_duration >= min_part_duration:\n",
        "      new_parts.append(sum(current_parts))\n",
        "      current_parts = []\n",
        "      current_duration = 0\n",
        "\n",
        "  return new_parts"
      ],
      "metadata": {
        "id": "C09zupqvMvVO"
      },
      "execution_count": 202,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def cut_part_in_half(part, min_duration=2, min_silence_len=10, silence_thresh=-50):\n",
        "  \"\"\"\n",
        "  Devide a given audio chunk to two almost equal subparts\n",
        "\n",
        "  Arguments:\n",
        "  - min_duration: min duration of resulting chunks\n",
        "  - min_silence_len: min silence len used to split audio on silent moments\n",
        "  - silence_thresh: silence threshold used to split audio on silent moments\n",
        "  \"\"\"\n",
        "\n",
        "  # Get total duration of audio\n",
        "  part_duration = part.duration_seconds\n",
        "\n",
        "  # Split the audio into subparts on silent moments that are candidate points for deviding the audio\n",
        "  subparts = split_on_silence(part, min_silence_len=min_silence_len, silence_thresh=silence_thresh, keep_silence=True)\n",
        "\n",
        "  # Duration of currently selected subparts to construct first half\n",
        "  current_duration = 0\n",
        "\n",
        "  for i, subpart in enumerate(subparts):\n",
        "    current_duration += subpart.duration_seconds\n",
        "\n",
        "    # Check if currently selected subparts have met half or original duration\n",
        "    if current_duration > part.duration_seconds / 2:\n",
        "\n",
        "      # Accept this division if the remaning of the audio is ok in terms of min_duration\n",
        "      if part_duration - current_duration > min_duration:\n",
        "        return [sum(subparts[:i + 1]), sum(subparts[i + 1:])]\n",
        "\n",
        "      # If selecting the last chunk has left less than min_duration for the second part,\n",
        "      # select chunks before it if min_duration is not violated\n",
        "      if current_duration - subpart.duration_seconds > min_duration:\n",
        "        return [sum(subparts[:i]), sum(subparts[i:])]\n",
        "\n",
        "      # If neither of natural (silence based) division above worked, split audio from middle moment\n",
        "      half = len(part) // 2\n",
        "      return [part[:half], part[half:]]"
      ],
      "metadata": {
        "id": "PzG4p0x3NSo2"
      },
      "execution_count": 203,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def split_part_to_max_duration(part, min_duration=2, max_duration=12, min_silence_len=10, silence_thresh=-50):\n",
        "  \"\"\"\n",
        "  Split the audio to smaller chunks by recursively dividing it to two almost equal supparts until all chunks are shorter than max_duration\n",
        "\n",
        "  Arguments:\n",
        "  - min_duration: minimum acceptable chunk duration\n",
        "  - max_duration: maximum acceptable chunk duration\n",
        "  - min_silence_len: min silence len used to split audio on silent moments\n",
        "  - silence_thresh: silence threshold used to split audio on silent moments\n",
        "  \"\"\"\n",
        "\n",
        "  # Return if input audio is already below max_duration\n",
        "  if part.duration_seconds <= max_duration:\n",
        "    return [part]\n",
        "\n",
        "  # Recursively divide the audio to two almost equal subparts otherwise\n",
        "  part1, part2 = cut_part_in_half(part, min_silence_len, silence_thresh)\n",
        "  return  split_part_to_max_duration(part1, min_duration, max_duration, min_silence_len, silence_thresh) + \\\n",
        "          split_part_to_max_duration(part2, min_duration, max_duration, min_silence_len, silence_thresh)"
      ],
      "metadata": {
        "id": "D7JaRfDDNVpW"
      },
      "execution_count": 204,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def split_parts_to_max_duration(parts, min_duration=2, max_duration=12, min_silence_len=10, silence_thresh=-50):\n",
        "  \"\"\"\n",
        "  Merely calls split_part_to_max_duration on a list of input audio and returns a list of audio chunks all below max_duration\n",
        "  \"\"\"\n",
        "\n",
        "  new_parts = []\n",
        "\n",
        "  for part in parts:\n",
        "    new_parts.extend(split_part_to_max_duration(part, min_duration, max_duration, min_silence_len, silence_thresh))\n",
        "\n",
        "  return new_parts"
      ],
      "metadata": {
        "id": "o3RRjKp_NYPL"
      },
      "execution_count": 205,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_audio_chunks(audio_file_path, min_chunk_duration=2, max_chunk_duration=12, min_silence_len_normal=150, silence_thresh=-50, min_silence_len_exessive=10):\n",
        "  \"\"\"\n",
        "  Splits the input audio file to chunks on silent moments, then merges the chunks to ensure a minimum duration\n",
        "  and breaks down large chunks to ensure a maximum duration\n",
        "\n",
        "  Arguments:\n",
        "  - audio_file_path: path to audio to be chunked\n",
        "  - min_chunk_duration: minimum acceptable chunk duration\n",
        "  - max_chunk_duration: maximum acceptable chunk duration\n",
        "  - min_silence_len_normal: min silence len used to split origianl audio to chunks at first attempt\n",
        "    More specifically, we first try to split the audio to chunks by silent moments that are longer and correspond to more natural stops.\n",
        "    Then relax this min duration to meet the min/max chunk durations with more risky silent moments.\n",
        "  - silence_thresh: silence threshold used to split audio on silent moments\n",
        "  - min_silence_len_exessive: min silence len used to split audio chunks further to ensure max_duration\n",
        "\n",
        "  Returns:\n",
        "  - a list of properly sized audio chunks splitted on natural silent moments whenever possible\n",
        "  \"\"\"\n",
        "\n",
        "  format = audio_file_path.split('.')[-1]\n",
        "  audio = AudioSegment.from_file(audio_file_path, format=format)\n",
        "\n",
        "  # Split on the more reliable and longer silent moments\n",
        "  parts = split_on_silence(audio, min_silence_len=min_silence_len_normal, silence_thresh=silence_thresh, keep_silence=True)\n",
        "\n",
        "  # Merge small audio chunks to ensure min_duration\n",
        "  parts_concat_to_min_duration = concat_parts_to_min_duration(parts, min_chunk_duration)\n",
        "\n",
        "  # Further split large chunks to ensure max_duration\n",
        "  parts_split_to_max_duration = split_parts_to_max_duration(parts_concat_to_min_duration, min_chunk_duration, max_chunk_duration, min_silence_len_exessive, silence_thresh)\n",
        "\n",
        "  return parts_split_to_max_duration"
      ],
      "metadata": {
        "id": "nvgxWN7FNbIb"
      },
      "execution_count": 206,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def match_transcript(transcript, text, max_lookup=500, max_removal=300, cer_threshold_1=0.05, cer_threshold_2=0.2, backtrack_cer=0.8, bactrack_search1_coeff=1.7, backtrack_search2_coeff=1.2, log_idx=0):\n",
        "  \"\"\"\n",
        "  Given a hypothesis transcript and the ground truth text, selects a subpart of the text that best matches the transcript.\n",
        "\n",
        "  Two search methods are used:\n",
        "  1- Interval Search: looks for all possible intervals like text[s:i]\n",
        "  2- Gapped Search: looks for all subparts formed as text[s:j] + text[k:i]\n",
        "\n",
        "  The first search is less computationally expensive, so getting a good result in this search is preferred.\n",
        "  In case no good match like text[i:j] is found, the second search method is run.\n",
        "\n",
        "  Arguments:\n",
        "  - transcript: the hypothesis (uncertain/asr output) transcript for which we want to find a matching text\n",
        "  - text: the ground truth text to look for the transctipt\n",
        "  - max_lookup: number of characters on which we search for matching texts\n",
        "  - max_removal: the maximum difference between j and k. i.e. the maximum gap size in second search\n",
        "  - cer_threshold_1: the CER value that causes and early stop of the search whenever reached\n",
        "  - cer_threshold_2: the CER value that prevents the Gapped Search if reached in Interval Search,\n",
        "    this value is also the maximum CER that is acceptable after the entire search process\n",
        "  - backtrack_cer: used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than this value,\n",
        "    it wont look for j and k indices\n",
        "  - bactrack_search1_coeff: also used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than\n",
        "   best cer in Interval Search multiplied by this value, it wont look for j and k indices\n",
        "  - bactrack_search2_coeff: also used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than\n",
        "   best cer in Gapped Search multiplied by this value, it wont look for j and k indices\n",
        "\n",
        "  min_cer, matching_text, end_pointer, \"REJECT\", time.time() - start_time, 2\n",
        "  Returns:\n",
        "  - min_cer: CER of the matching text found\n",
        "  - matching_text: the text subpart selected from the input text as the best matching transcript\n",
        "  - end_pointer: pointer indicating how much we have proceeded in the input text already to match with audio files. Used in the upper function call.\n",
        "  - quality: a quality level either \"HIGH\" for cer values below cer_threshold_1, \"MIDDLE\" for cer values between cer_threshold_1 and cer_threshold_2, and \"REJECT\" for ver values more than cer_threshold_2\n",
        "  - time: processing time of the input audio chunk alignment\n",
        "  \"\"\"\n",
        "\n",
        "  start_time = time.time()\n",
        "\n",
        "  min_cer = float('inf')\n",
        "  matching_text = ''\n",
        "  end_pointer = -1\n",
        "\n",
        "  print(f\"\\tTranscript:\\t\\t {transcript}\")\n",
        "\n",
        "  # Interval Search: search on text[s:i] subparts\n",
        "  for s in range(min(len(text), max_removal)):\n",
        "\n",
        "    # Ensures the reference text is split in word boundaries only\n",
        "    if s != 0 and not (match_special_character(text[s - 1]) and not match_special_character(text[s])): continue\n",
        "\n",
        "    for i in range(s + 1, min(len(text), s + 1 + max_lookup)):\n",
        "\n",
        "        # Ensures the reference text is split in word boundaries only\n",
        "        if not i == len(text) and not (match_special_character(text[i - 1]) and not match_special_character(text[i])): continue\n",
        "\n",
        "        cer_value = get_texts_cer(text[s:i], transcript)\n",
        "\n",
        "       # Save best search results\n",
        "        if cer_value < min_cer:\n",
        "            min_cer = cer_value\n",
        "            end_pointer = i\n",
        "            matching_text = text[s:i]\n",
        "\n",
        "        # Early stop the search if CER value below cer_threshold_1\n",
        "        if cer_value <= cer_threshold_1:\n",
        "            print(f\"\\033[92m\\tInterval Search: Matching Text:{matching_text}\\033[0m\")\n",
        "            print(f\"\\033[92m\\tInterval Search: CER: {min_cer}, End Pointer: {end_pointer}\\033[0m\")\n",
        "            return min_cer, matching_text, end_pointer, \"HIGH\", 1\n",
        "\n",
        "  # Dont proceed to Gapped Search if the CER value from Interval Search is below cer_threshold_2\n",
        "  if min_cer <= cer_threshold_2:\n",
        "    print(f\"\\033[93m\\tInterval Search: Matching Text: {matching_text}\\033[0m\")\n",
        "    print(f\"\\033[93m\\tInterval Search: CER: {min_cer}, End Pointer: {end_pointer}\\033[0m\")\n",
        "    return min_cer, matching_text, end_pointer, \"MIDDLE\", 1\n",
        "\n",
        "  # If Interval Search not accepted, just log the best results\n",
        "  print(f\"\\tInterval Search: Matching Text: {matching_text}\")\n",
        "  print(f\"\\tInterval Search: CER: {min_cer}, End Pointer: {end_pointer}\")\n",
        "  interval_search_cer = min_cer\n",
        "\n",
        "\n",
        "  # Gapped Search: search on text[s:k] + text[j:i] subparts\n",
        "  gapped_search_cer = float('inf')\n",
        "  for s in range(min(len(text), max_removal)):\n",
        "\n",
        "    # Ensures the reference text is split in word boundaries only\n",
        "    if s != 0 and not (match_special_character(text[s - 1]) and not match_special_character(text[s])): continue\n",
        "\n",
        "    for i in range(s + 1, min(len(text) + 1, s + 1 + max_lookup)):\n",
        "\n",
        "      # Ensures the reference text is split in word boundaries only\n",
        "      if not i == len(text) and not (match_special_character(text[i - 1]) and not match_special_character(text[i])): continue\n",
        "\n",
        "      # Back track from searching j and k if CER value above the defined thresholds\n",
        "      interval_cer = get_texts_cer(text[s:i], transcript)\n",
        "      if interval_cer > backtrack_cer or\\\n",
        "        interval_cer > interval_search_cer * bactrack_search1_coeff or \\\n",
        "        interval_cer > gapped_search_cer * backtrack_search2_coeff: continue\n",
        "\n",
        "      for j in range(s + 1, i):\n",
        "\n",
        "        # Ensures the reference text is split in word boundaries only\n",
        "        if not (match_special_character(text[j - 1]) and not match_special_character(text[j])): continue\n",
        "\n",
        "        for k in range(j + 1, min(j + max_removal, i)):\n",
        "\n",
        "          # Ensures the reference text is split in word boundaries only\n",
        "          if not (match_special_character(text[k - 1]) and not match_special_character(text[k])): continue\n",
        "\n",
        "          cer_value = get_texts_cer(text[s:j] + text[k:i], transcript)\n",
        "\n",
        "          # Update Gapped Search best results used for backtracking\n",
        "          gapped_search_cer = min(gapped_search_cer, cer_value)\n",
        "\n",
        "          # Save best results of entire search\n",
        "          if cer_value < min_cer:\n",
        "              min_cer = cer_value\n",
        "              end_pointer = i\n",
        "              matching_text = text[s:j] + text[k:i]\n",
        "\n",
        "          # Early stop the search if CER value below cer_threshold_1\n",
        "          if cer_value <= cer_threshold_1:\n",
        "              print(f\"\\033[92m\\tGapped Search: Matching Text:{matching_text}\\033[0m\")\n",
        "              print(f\"\\033[92m\\tGapped Search: CER: {min_cer}, End Pointer: {end_pointer}\\033[0m\")\n",
        "              return min_cer, matching_text, end_pointer, \"HIGH\", 2\n",
        "\n",
        "          if time.time() - start_time > 900:\n",
        "            print(\"Search timeout. Quitting...\")\n",
        "            return min_cer, matching_text, end_pointer, \"REJECT\", 0\n",
        "\n",
        "  # Accept the the result with \"MIDDLE\" quality if the CER value from Interval Search is below cer_threshold_2\n",
        "  if min_cer <= cer_threshold_2:\n",
        "    print(f\"\\033[93m\\tBoth Searches: Matching Text:{matching_text}\\033[0m\")\n",
        "    print(f\"\\033[93m\\tBoth Searches: CER: {min_cer}, End Pointer: {end_pointer}\\033[0m\")\n",
        "\n",
        "    return min_cer, matching_text, end_pointer, \"MIDDLE\", 2\n",
        "\n",
        "\n",
        "  # Return with \"REJECT\" status meaning no good matching text found\n",
        "  print(f\"\\033[91m\\tBoth Searches: Matching Text:{matching_text}\\033[0m\")\n",
        "  print(f\"\\033[91m\\tBoth Searches: CER: {min_cer}, End Pointer: {end_pointer}\\033[0m\")\n",
        "  return min_cer, matching_text, end_pointer, \"REJECT\", 2"
      ],
      "metadata": {
        "id": "MANQ9TFRNlkg"
      },
      "execution_count": 207,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_word_count(text):\n",
        "    words = word_tokenize(text)\n",
        "    valid_words = [word for word in words if word_pattern.match(word)]\n",
        "    return len(valid_words)"
      ],
      "metadata": {
        "id": "DBTBtVw5NrJZ"
      },
      "execution_count": 208,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def match_audio(audio, output_chunk_path, text, max_lookup=500, max_removal=300, cer_threshold_1=0.05, cer_threshold_2=0.2, backtrack_cer=0.8, bactrack_search1_coeff=1.7, backtrack_search2_coeff=1.2, log_idx=0):\n",
        "    \"\"\"\n",
        "      Given an audio chunk and the ground truth text, selects a subpart of the text that best matches the audio.\n",
        "      It tries the best transcripts from asr models in order until one is accepted.\n",
        "\n",
        "      Arguments:\n",
        "      - audio: audio segment to for which we want to find a matching text\n",
        "      - text: the ground truth text to look for the transctipt\n",
        "      - max_lookup: number of characters on which we search for matching texts\n",
        "      - max_removal: the maximum difference between j and k. i.e. the maximum gap size in second search\n",
        "      - cer_threshold_1: the CER value that causes and early stop of the search whenever reached\n",
        "      - cer_threshold_2: the CER value that prevents the Gapped Search if reached in Interval Search,\n",
        "        this value is also the maximum CER that is acceptable after the entire search process\n",
        "      - backtrack_cer: used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than this value,\n",
        "        it wont look for j and k indices\n",
        "      - bactrack_search1_coeff: also used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than\n",
        "      best cer in Interval Search multiplied by this value, it wont look for j and k indices\n",
        "      - bactrack_search2_coeff: also used to backtrack some combinations in the Gapped Search. if text[s:i] is already more than\n",
        "      best cer in Gapped Search multiplied by this value, it wont look for j and k indices\n",
        "\n",
        "      Returns:\n",
        "      - min_cer: CER of the matching text found\n",
        "      - transcript: the hypothesis transcript from ASR for the given audio chunk\n",
        "      - matching_text: the text subpart selected from the input text as the best matching transcript\n",
        "      - end_pointer: pointer indicating how much we have proceeded in the input text already to match with audio files. Used in the upper function call.\n",
        "      - quality: a quality level either \"HIGH\" for cer values below cer_threshold_1, \"MIDDLE\" for cer values between cer_threshold_1 and cer_threshold_2, and \"REJECT\" for ver values more than cer_threshold_2\n",
        "      - time: processing time of the input audio chunk alignment\n",
        "      - asrs: the asr models that were used in order\n",
        "    \"\"\"\n",
        "\n",
        "    start_time = time.time()\n",
        "\n",
        "    tempfile = f\"temp_{log_idx}.wav\"\n",
        "    audio.export(tempfile, format=\"wav\")\n",
        "    transcripts = get_best_transcripts(tempfile, output_chunk_path, log=True)\n",
        "    os.remove(tempfile)\n",
        "\n",
        "    if not transcripts:\n",
        "       return float('inf'), '', '', 0, -1, 'REJECT', -1, time.time() - start_time, []\n",
        "\n",
        "    asrs = []\n",
        "    for asr, transcript in transcripts:\n",
        "      min_cer, matching_text, end_pointer, status, search_type = match_transcript(transcript, text, max_lookup, max_removal, cer_threshold_1, cer_threshold_2, backtrack_cer, bactrack_search1_coeff, backtrack_search2_coeff, log_idx)\n",
        "      asrs.append(asr)\n",
        "      if status != 'REJECT': break\n",
        "\n",
        "    return min_cer, transcript, matching_text, get_word_count(matching_text), end_pointer, status, search_type, time.time() - start_time, asrs\n"
      ],
      "metadata": {
        "id": "1X5tDMEmNt81"
      },
      "execution_count": 209,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def forced_align(audio_file_path, text_file_path, alignment_dir, metadata, processed_files, log_idx):\n",
        "  \"\"\"\n",
        "  Given an input audio file path and a ground truth text file path, splits the audio into chunks and searches for matching subparts form the text for each chunk\n",
        "  It exports each audio chunk and its matching text as files to the alignment dir and writes its results to a metadata file given as input.\n",
        "  \"\"\"\n",
        "  # Get file name used for logging\n",
        "  audio_filename = audio_file_path.split('/')[-1].split('.')[-2]\n",
        "\n",
        "  # Create a directory for aligned chunk files of the audio\n",
        "  audio_alignment_dir = os.path.join(alignment_dir, audio_filename)\n",
        "\n",
        "  # Stop if aleady aligned\n",
        "  if audio_filename in processed_files:\n",
        "     print(f\"Skipping file {audio_filename}.wav. Forced alignment directory exists.\")\n",
        "     return audio_alignment_dir\n",
        "\n",
        "  os.makedirs(audio_alignment_dir, exist_ok=True)\n",
        "\n",
        "  text = ''\n",
        "  with open(text_file_path, 'r') as f: text = f.read()\n",
        "  audio_chunks = get_audio_chunks(audio_file_path)\n",
        "\n",
        "  text_pointer = 0\n",
        "  max_lookup = 500  # number of characters on which we search for matching texts\n",
        "  max_lookup_stepsize = 500  # constant increase value to max_lookup to recover from previously unmatched chracters in reference text\n",
        "\n",
        "  for i, chunk in enumerate(audio_chunks, start=1):\n",
        "    print(f\"Matching chunk {i}/{len(audio_chunks)} of {audio_filename}.wav...\")\n",
        "\n",
        "    chunk_audio_path = os.path.join(audio_alignment_dir, f'{audio_filename}-{i}.wav')\n",
        "\n",
        "    # Search for best matching text from reference text\n",
        "    min_cer, transcript, matching_text, matching_text_word_count, pointer_shift, quality, search_type, processing_time, asrs = match_audio(chunk, chunk_audio_path, text[text_pointer:], max_lookup, log_idx=log_idx)\n",
        "\n",
        "    # Write search results to metadata\n",
        "    with open(metadata, 'a', newline='') as csvfile:\n",
        "      writer = csv.writer(csvfile)\n",
        "      writer.writerow([audio_alignment_dir, f'{audio_filename}-{i}.wav', matching_text, matching_text_word_count, chunk.duration_seconds, 0, min_cer, quality, transcript, processing_time, search_type, asrs])\n",
        "\n",
        "    # Export the audio chunk\n",
        "    print(f\"\\tWriting audio chunk {i}...\")\n",
        "    chunk.export(chunk_audio_path, format=\"wav\")\n",
        "\n",
        "    if quality == \"REJECT\":\n",
        "      # If no matching text was found, there might be some leftover text at start.\n",
        "      # We increase max_lookup to recover from this mismatch\n",
        "      max_lookup += max_lookup_stepsize\n",
        "\n",
        "    else:\n",
        "      # If a matching text was found, simply move the starting pointer of the reference text and reset max_lookup to normal\n",
        "      max_lookup = max_lookup_stepsize\n",
        "      text_pointer += pointer_shift - 15  # Let about one word be mistakenly included in last match\n",
        "\n",
        "\n",
        "    # Export the matching text to a file\n",
        "    print(f\"\\tWriting text chunk {i}...\")\n",
        "    segment_text_path = os.path.join(audio_alignment_dir, f\"{audio_filename}-{i}.txt\")\n",
        "    with open(segment_text_path, 'w', encoding='utf-8') as segment_file:\n",
        "        segment_file.write(matching_text.strip())\n",
        "\n",
        "  return audio_alignment_dir"
      ],
      "metadata": {
        "id": "qfo4wDbwMmk-"
      },
      "execution_count": 210,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Process Data\n",
        "Here we define the full processing pipeline. It contains three main steps:\n",
        "\n",
        "\n",
        "\n",
        "1.   **Pre-processing:** Format conversion, background music removal, and text pre-processing\n",
        "2.   **Alignment:** Start-end alignment and forced alignment\n",
        "3.   **Post-processing:** Silence removal and stereo to mono\n",
        "\n"
      ],
      "metadata": {
        "id": "f3utCBZIRpCh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import shutil\n",
        "import csv\n",
        "import argparse"
      ],
      "metadata": {
        "id": "8h8WD5ntSVFe"
      },
      "execution_count": 211,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def pre_process(input_audio_path, input_text_path, output_dir_path, log_idx):\n",
        "    # pre-process audio\n",
        "    output_audio_file_path = convert_mp3_to_wav(input_audio_path, output_dir_path)\n",
        "    output_audio_file_path = remove_background_music(output_audio_file_path, output_dir_path, log_idx=log_idx)\n",
        "\n",
        "    # pre-process text\n",
        "    output_text_file_path = process_text(input_text_path, output_dir_path)\n",
        "\n",
        "    return output_audio_file_path, output_text_file_path"
      ],
      "metadata": {
        "id": "SoPIFjcQSaxN"
      },
      "execution_count": 212,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def align(input_audio_path, input_text_path, output_start_end_align_dir_path, output_forced_align_dir_path, metadata, force_aligned_files, log_idx):\n",
        "    output_audio_file_path, output_text_file_path = start_end_align(input_audio_path, input_text_path, output_start_end_align_dir_path, log_idx=log_idx)\n",
        "    audio_forced_align_dir_path = forced_align(output_audio_file_path, output_text_file_path, output_forced_align_dir_path, metadata, force_aligned_files, log_idx=log_idx)\n",
        "\n",
        "    return audio_forced_align_dir_path"
      ],
      "metadata": {
        "id": "g6jhxLq6ScOM"
      },
      "execution_count": 213,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def post_process(forced_aligned_dir_path, post_processed_dir_path, metadata):\n",
        "   for filename in os.listdir(forced_aligned_dir_path):\n",
        "        src_path = os.path.join(forced_aligned_dir_path, filename)\n",
        "\n",
        "        if filename.endswith(\".txt\"):   # Copy text files exactly\n",
        "            dest_path = os.path.join(post_processed_dir_path, filename)\n",
        "            shutil.copyfile(src_path, dest_path)\n",
        "\n",
        "        if filename.endswith(\".wav\"):\n",
        "            output_file_path = remove_silent_parts(src_path, post_processed_dir_path, metadata)\n",
        "            convert_stereo_to_mono(output_file_path, post_processed_dir_path)"
      ],
      "metadata": {
        "id": "XcTPT-ioSi_7"
      },
      "execution_count": 214,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def process_data(data_root, stage_dirs, metadata, force_aligned_files_path, range_start=None, range_end=None, log_idx=None):\n",
        "    # Read the forced aligned audio files as a set\n",
        "    force_aligned_files = None\n",
        "    with open(force_aligned_files_path, 'r') as f:\n",
        "      force_aligned_files = set([line.strip() for line in f.readlines()])\n",
        "\n",
        "\n",
        "    # Get total number of audio files in the source directory\n",
        "    audio_files = [audio_file for audio_file in os.listdir(data_root) if audio_file.endswith('.mp3')]\n",
        "\n",
        "\n",
        "    # Filter the audio files if the processing is limited to some range\n",
        "    if range_start is not None and range_end is not None:\n",
        "        audio_files = [audio_file for audio_file in audio_files if range_start <= int(audio_file.split('.')[0]) < range_end]\n",
        "\n",
        "\n",
        "    total_files = len(audio_files)\n",
        "\n",
        "\n",
        "    # Iterate through all audio files in the source directory\n",
        "    for idx, audio_file in enumerate(audio_files, start=1):\n",
        "        progress = f\"({idx}/{total_files})\"\n",
        "        print(f\"{progress}: Processing file {audio_file}...\")\n",
        "\n",
        "\n",
        "        # Get corresponding text file\n",
        "        audio_file_path = os.path.join(data_root, audio_file)\n",
        "        audio_file_name = audio_file.split('.')[0]\n",
        "        text_file_path = os.path.join(data_root, audio_file_name + '.txt')\n",
        "\n",
        "\n",
        "        # Pass the raw data through pipeline\n",
        "        audio_path, text_path = pre_process(audio_file_path, text_file_path, stage_dirs[0], log_idx=log_idx)\n",
        "\n",
        "        force_alignment_path = align(audio_path, text_path, stage_dirs[1], stage_dirs[2], metadata, force_aligned_files, log_idx=log_idx)\n",
        "        if audio_file_name not in force_aligned_files:\n",
        "            with open(force_aligned_files_path, 'a') as f: f.writelines(audio_file_name + '\\n')\n",
        "\n",
        "        post_process(force_alignment_path, stage_dirs[3], metadata)"
      ],
      "metadata": {
        "id": "qywRAJ-eSlis"
      },
      "execution_count": 215,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run the Processing Pipeline\n",
        "Four directories are defined as stage dirs. The results of the data at different stages of the pipeline, including the pre-processed data, the start-end aligned data, the forced-aligned data, and the post-processed data area stored in these dirs.\n",
        "\n",
        "The arguments `start`, `end`, and `log` are used for parallel processing of data. You defined the range of files to process in each thread by setting the `start` and `end` index of the files and label that run with a number for the `log` argument. This `log` number will be used to avoid conflicts in naming temporary and log files. If you are running a single thread, simply set `start` to 1, `end` to the index of the last file, and `log` to an arbitrary number like 1.\n",
        "\n",
        "NOTE: we assume the raw audio and text files have numeric names as in ManaTTS data."
      ],
      "metadata": {
        "id": "18GBvf7YSsB4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # Get file processing range if provided\n",
        "# parser = argparse.ArgumentParser(description=\"Process files within a specified range.\")\n",
        "# parser.add_argument(\"--start\", type=int, default=None, help=\"Start range value (inclusive)\")\n",
        "# parser.add_argument(\"--end\", type=int, default=None, help=\"End range value (exclusive)\")\n",
        "# parser.add_argument(\"--log\", type=int, default=None, help=\"Log files suffix\")\n",
        "# args = parser.parse_args()\n",
        "# start = args.start\n",
        "# end = args.end\n",
        "# log = args.log\n",
        "\n",
        "start = 1\n",
        "end = 600\n",
        "log = 1\n",
        "\n",
        "data_root = 'raw'\n",
        "\n",
        "# Define directories to save different states of the processing data\n",
        "stage_dirs = [\n",
        "    'pre-processed',\n",
        "    'start-end-alignemnt',\n",
        "    'forced-alignment',\n",
        "    'post-processed'\n",
        "]\n",
        "for stage_dir in stage_dirs: os.makedirs(stage_dir, exist_ok=True)\n",
        "\n",
        "\n",
        "# Since forced alignment is computaionally expensive, we will keep track of aligned audio in a file\n",
        "force_aligned_files_path = f'force_aligned_files_path-{log}.txt'\n",
        "if not os.path.exists(force_aligned_files_path): open(force_aligned_files_path, 'a').close()\n",
        "\n",
        "\n",
        "metadata = f'metadata-{log}.csv'\n",
        "\n",
        "# Create the metadata file with header\n",
        "if not os.path.exists(metadata):\n",
        "  with open(metadata, 'w', newline='') as csvfile:\n",
        "    writer = csv.writer(csvfile)\n",
        "    writer.writerow(['Path', 'Audio', 'Transcript', 'Transcript Word Count', 'Duration', 'Silence Removed Duration', 'CER', 'Status', 'Hypothesis', 'Processing Time', 'Search Type', 'ASRs'])\n",
        "\n",
        "\n",
        "process_data(data_root, stage_dirs, metadata, force_aligned_files_path, start, end, log)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eti-Th39SK77",
        "outputId": "9a28dc0b-aa4d-4cab-844b-37a1f5553998"
      },
      "execution_count": 217,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1/2): Processing file 4.mp3...\n",
            "Skipping file 4.wav. Vocals file exists.\n",
            "Skipping file 4.txt. Processed text file already exists.\n",
            "Skipping file 4.wav. Start-End aligned audio exists.\n",
            "Skipping file 4.wav. Forced alignment directory exists.\n",
            "Skipping file 4-170.wav. Silence removed file exists.\n",
            "Skipping file 4-43.wav. Silence removed file exists.\n",
            "Skipping file 4-77.wav. Silence removed file exists.\n",
            "Skipping file 4-87.wav. Silence removed file exists.\n",
            "Skipping file 4-184.wav. Silence removed file exists.\n",
            "Skipping file 4-38.wav. Silence removed file exists.\n",
            "Skipping file 4-60.wav. Silence removed file exists.\n",
            "Skipping file 4-203.wav. Silence removed file exists.\n",
            "Skipping file 4-94.wav. Silence removed file exists.\n",
            "Skipping file 4-116.wav. Silence removed file exists.\n",
            "Skipping file 4-136.wav. Silence removed file exists.\n",
            "Skipping file 4-130.wav. Silence removed file exists.\n",
            "Skipping file 4-219.wav. Silence removed file exists.\n",
            "Skipping file 4-2.wav. Silence removed file exists.\n",
            "Skipping file 4-208.wav. Silence removed file exists.\n",
            "Skipping file 4-157.wav. Silence removed file exists.\n",
            "Skipping file 4-80.wav. Silence removed file exists.\n",
            "Skipping file 4-123.wav. Silence removed file exists.\n",
            "Skipping file 4-188.wav. Silence removed file exists.\n",
            "Skipping file 4-101.wav. Silence removed file exists.\n",
            "Skipping file 4-10.wav. Silence removed file exists.\n",
            "Skipping file 4-132.wav. Silence removed file exists.\n",
            "Skipping file 4-114.wav. Silence removed file exists.\n",
            "Skipping file 4-133.wav. Silence removed file exists.\n",
            "Skipping file 4-150.wav. Silence removed file exists.\n",
            "Skipping file 4-182.wav. Silence removed file exists.\n",
            "Skipping file 4-113.wav. Silence removed file exists.\n",
            "Skipping file 4-96.wav. Silence removed file exists.\n",
            "Skipping file 4-6.wav. Silence removed file exists.\n",
            "Skipping file 4-187.wav. Silence removed file exists.\n",
            "Skipping file 4-67.wav. Silence removed file exists.\n",
            "Skipping file 4-129.wav. Silence removed file exists.\n",
            "Skipping file 4-53.wav. Silence removed file exists.\n",
            "Skipping file 4-47.wav. Silence removed file exists.\n",
            "Skipping file 4-31.wav. Silence removed file exists.\n",
            "Skipping file 4-138.wav. Silence removed file exists.\n",
            "Skipping file 4-92.wav. Silence removed file exists.\n",
            "Skipping file 4-180.wav. Silence removed file exists.\n",
            "Skipping file 4-222.wav. Silence removed file exists.\n",
            "Skipping file 4-73.wav. Silence removed file exists.\n",
            "Skipping file 4-179.wav. Silence removed file exists.\n",
            "Skipping file 4-126.wav. Silence removed file exists.\n",
            "Skipping file 4-85.wav. Silence removed file exists.\n",
            "Skipping file 4-39.wav. Silence removed file exists.\n",
            "Skipping file 4-58.wav. Silence removed file exists.\n",
            "Skipping file 4-91.wav. Silence removed file exists.\n",
            "Skipping file 4-21.wav. Silence removed file exists.\n",
            "Skipping file 4-197.wav. Silence removed file exists.\n",
            "Skipping file 4-82.wav. Silence removed file exists.\n",
            "Skipping file 4-105.wav. Silence removed file exists.\n",
            "Skipping file 4-66.wav. Silence removed file exists.\n",
            "Skipping file 4-196.wav. Silence removed file exists.\n",
            "Skipping file 4-13.wav. Silence removed file exists.\n",
            "Skipping file 4-41.wav. Silence removed file exists.\n",
            "Skipping file 4-14.wav. Silence removed file exists.\n",
            "Skipping file 4-100.wav. Silence removed file exists.\n",
            "Skipping file 4-88.wav. Silence removed file exists.\n",
            "Skipping file 4-32.wav. Silence removed file exists.\n",
            "Skipping file 4-72.wav. Silence removed file exists.\n",
            "Skipping file 4-115.wav. Silence removed file exists.\n",
            "Skipping file 4-152.wav. Silence removed file exists.\n",
            "Skipping file 4-49.wav. Silence removed file exists.\n",
            "Skipping file 4-210.wav. Silence removed file exists.\n",
            "Skipping file 4-148.wav. Silence removed file exists.\n",
            "Skipping file 4-22.wav. Silence removed file exists.\n",
            "Skipping file 4-112.wav. Silence removed file exists.\n",
            "Skipping file 4-153.wav. Silence removed file exists.\n",
            "Skipping file 4-59.wav. Silence removed file exists.\n",
            "Skipping file 4-158.wav. Silence removed file exists.\n",
            "Skipping file 4-110.wav. Silence removed file exists.\n",
            "Skipping file 4-181.wav. Silence removed file exists.\n",
            "Skipping file 4-99.wav. Silence removed file exists.\n",
            "Skipping file 4-9.wav. Silence removed file exists.\n",
            "Skipping file 4-86.wav. Silence removed file exists.\n",
            "Skipping file 4-183.wav. Silence removed file exists.\n",
            "Skipping file 4-55.wav. Silence removed file exists.\n",
            "Skipping file 4-102.wav. Silence removed file exists.\n",
            "Skipping file 4-75.wav. Silence removed file exists.\n",
            "Skipping file 4-194.wav. Silence removed file exists.\n",
            "Skipping file 4-25.wav. Silence removed file exists.\n",
            "Skipping file 4-29.wav. Silence removed file exists.\n",
            "Skipping file 4-131.wav. Silence removed file exists.\n",
            "Skipping file 4-159.wav. Silence removed file exists.\n",
            "Skipping file 4-221.wav. Silence removed file exists.\n",
            "Skipping file 4-16.wav. Silence removed file exists.\n",
            "Skipping file 4-193.wav. Silence removed file exists.\n",
            "Skipping file 4-146.wav. Silence removed file exists.\n",
            "Skipping file 4-93.wav. Silence removed file exists.\n",
            "Skipping file 4-213.wav. Silence removed file exists.\n",
            "Skipping file 4-206.wav. Silence removed file exists.\n",
            "Skipping file 4-64.wav. Silence removed file exists.\n",
            "Skipping file 4-144.wav. Silence removed file exists.\n",
            "Skipping file 4-127.wav. Silence removed file exists.\n",
            "Skipping file 4-28.wav. Silence removed file exists.\n",
            "Skipping file 4-142.wav. Silence removed file exists.\n",
            "Skipping file 4-124.wav. Silence removed file exists.\n",
            "Skipping file 4-69.wav. Silence removed file exists.\n",
            "Skipping file 4-11.wav. Silence removed file exists.\n",
            "Skipping file 4-185.wav. Silence removed file exists.\n",
            "Skipping file 4-104.wav. Silence removed file exists.\n",
            "Skipping file 4-189.wav. Silence removed file exists.\n",
            "Skipping file 4-166.wav. Silence removed file exists.\n",
            "Skipping file 4-12.wav. Silence removed file exists.\n",
            "Skipping file 4-202.wav. Silence removed file exists.\n",
            "Skipping file 4-120.wav. Silence removed file exists.\n",
            "Skipping file 4-140.wav. Silence removed file exists.\n",
            "Skipping file 4-141.wav. Silence removed file exists.\n",
            "Skipping file 4-165.wav. Silence removed file exists.\n",
            "Skipping file 4-164.wav. Silence removed file exists.\n",
            "Skipping file 4-30.wav. Silence removed file exists.\n",
            "Skipping file 4-20.wav. Silence removed file exists.\n",
            "Skipping file 4-168.wav. Silence removed file exists.\n",
            "Skipping file 4-209.wav. Silence removed file exists.\n",
            "Skipping file 4-98.wav. Silence removed file exists.\n",
            "Skipping file 4-111.wav. Silence removed file exists.\n",
            "Skipping file 4-40.wav. Silence removed file exists.\n",
            "Skipping file 4-134.wav. Silence removed file exists.\n",
            "Skipping file 4-63.wav. Silence removed file exists.\n",
            "Skipping file 4-5.wav. Silence removed file exists.\n",
            "Skipping file 4-151.wav. Silence removed file exists.\n",
            "Skipping file 4-81.wav. Silence removed file exists.\n",
            "Skipping file 4-1.wav. Silence removed file exists.\n",
            "Skipping file 4-160.wav. Silence removed file exists.\n",
            "Skipping file 4-122.wav. Silence removed file exists.\n",
            "Skipping file 4-19.wav. Silence removed file exists.\n",
            "Skipping file 4-195.wav. Silence removed file exists.\n",
            "Skipping file 4-207.wav. Silence removed file exists.\n",
            "Skipping file 4-84.wav. Silence removed file exists.\n",
            "Skipping file 4-33.wav. Silence removed file exists.\n",
            "Skipping file 4-57.wav. Silence removed file exists.\n",
            "Skipping file 4-174.wav. Silence removed file exists.\n",
            "Skipping file 4-107.wav. Silence removed file exists.\n",
            "Skipping file 4-190.wav. Silence removed file exists.\n",
            "Skipping file 4-65.wav. Silence removed file exists.\n",
            "Skipping file 4-68.wav. Silence removed file exists.\n",
            "Skipping file 4-177.wav. Silence removed file exists.\n",
            "Skipping file 4-37.wav. Silence removed file exists.\n",
            "Skipping file 4-27.wav. Silence removed file exists.\n",
            "Skipping file 4-167.wav. Silence removed file exists.\n",
            "Skipping file 4-44.wav. Silence removed file exists.\n",
            "Skipping file 4-155.wav. Silence removed file exists.\n",
            "Skipping file 4-119.wav. Silence removed file exists.\n",
            "Skipping file 4-95.wav. Silence removed file exists.\n",
            "Skipping file 4-223.wav. Silence removed file exists.\n",
            "Skipping file 4-143.wav. Silence removed file exists.\n",
            "Skipping file 4-154.wav. Silence removed file exists.\n",
            "Skipping file 4-74.wav. Silence removed file exists.\n",
            "Skipping file 4-212.wav. Silence removed file exists.\n",
            "Skipping file 4-214.wav. Silence removed file exists.\n",
            "Skipping file 4-71.wav. Silence removed file exists.\n",
            "Skipping file 4-121.wav. Silence removed file exists.\n",
            "Skipping file 4-178.wav. Silence removed file exists.\n",
            "Skipping file 4-211.wav. Silence removed file exists.\n",
            "Skipping file 4-50.wav. Silence removed file exists.\n",
            "Skipping file 4-90.wav. Silence removed file exists.\n",
            "Skipping file 4-45.wav. Silence removed file exists.\n",
            "Skipping file 4-8.wav. Silence removed file exists.\n",
            "Skipping file 4-52.wav. Silence removed file exists.\n",
            "Skipping file 4-137.wav. Silence removed file exists.\n",
            "Skipping file 4-173.wav. Silence removed file exists.\n",
            "Skipping file 4-4.wav. Silence removed file exists.\n",
            "Skipping file 4-46.wav. Silence removed file exists.\n",
            "Skipping file 4-62.wav. Silence removed file exists.\n",
            "Skipping file 4-125.wav. Silence removed file exists.\n",
            "Skipping file 4-145.wav. Silence removed file exists.\n",
            "Skipping file 4-89.wav. Silence removed file exists.\n",
            "Skipping file 4-23.wav. Silence removed file exists.\n",
            "Skipping file 4-162.wav. Silence removed file exists.\n",
            "Skipping file 4-204.wav. Silence removed file exists.\n",
            "Skipping file 4-128.wav. Silence removed file exists.\n",
            "Skipping file 4-42.wav. Silence removed file exists.\n",
            "Skipping file 4-186.wav. Silence removed file exists.\n",
            "Skipping file 4-156.wav. Silence removed file exists.\n",
            "Skipping file 4-61.wav. Silence removed file exists.\n",
            "Skipping file 4-103.wav. Silence removed file exists.\n",
            "Skipping file 4-163.wav. Silence removed file exists.\n",
            "Skipping file 4-217.wav. Silence removed file exists.\n",
            "Skipping file 4-78.wav. Silence removed file exists.\n",
            "Skipping file 4-139.wav. Silence removed file exists.\n",
            "Skipping file 4-215.wav. Silence removed file exists.\n",
            "Skipping file 4-76.wav. Silence removed file exists.\n",
            "Skipping file 4-172.wav. Silence removed file exists.\n",
            "Skipping file 4-149.wav. Silence removed file exists.\n",
            "Skipping file 4-169.wav. Silence removed file exists.\n",
            "Skipping file 4-117.wav. Silence removed file exists.\n",
            "Skipping file 4-3.wav. Silence removed file exists.\n",
            "Skipping file 4-54.wav. Silence removed file exists.\n",
            "Skipping file 4-109.wav. Silence removed file exists.\n",
            "Skipping file 4-200.wav. Silence removed file exists.\n",
            "Skipping file 4-18.wav. Silence removed file exists.\n",
            "Skipping file 4-191.wav. Silence removed file exists.\n",
            "Skipping file 4-175.wav. Silence removed file exists.\n",
            "Skipping file 4-147.wav. Silence removed file exists.\n",
            "Skipping file 4-26.wav. Silence removed file exists.\n",
            "Skipping file 4-48.wav. Silence removed file exists.\n",
            "Skipping file 4-7.wav. Silence removed file exists.\n",
            "Skipping file 4-15.wav. Silence removed file exists.\n",
            "Skipping file 4-83.wav. Silence removed file exists.\n",
            "Skipping file 4-79.wav. Silence removed file exists.\n",
            "Skipping file 4-192.wav. Silence removed file exists.\n",
            "Skipping file 4-135.wav. Silence removed file exists.\n",
            "Skipping file 4-106.wav. Silence removed file exists.\n",
            "Skipping file 4-161.wav. Silence removed file exists.\n",
            "Skipping file 4-17.wav. Silence removed file exists.\n",
            "Skipping file 4-171.wav. Silence removed file exists.\n",
            "Skipping file 4-56.wav. Silence removed file exists.\n",
            "Skipping file 4-34.wav. Silence removed file exists.\n",
            "Skipping file 4-51.wav. Silence removed file exists.\n",
            "Skipping file 4-108.wav. Silence removed file exists.\n",
            "Skipping file 4-97.wav. Silence removed file exists.\n",
            "Skipping file 4-216.wav. Silence removed file exists.\n",
            "Skipping file 4-205.wav. Silence removed file exists.\n",
            "Skipping file 4-201.wav. Silence removed file exists.\n",
            "Skipping file 4-198.wav. Silence removed file exists.\n",
            "Skipping file 4-70.wav. Silence removed file exists.\n",
            "Skipping file 4-176.wav. Silence removed file exists.\n",
            "Skipping file 4-118.wav. Silence removed file exists.\n",
            "Skipping file 4-218.wav. Silence removed file exists.\n",
            "Skipping file 4-199.wav. Silence removed file exists.\n",
            "Skipping file 4-35.wav. Silence removed file exists.\n",
            "Skipping file 4-220.wav. Silence removed file exists.\n",
            "Skipping file 4-36.wav. Silence removed file exists.\n",
            "Skipping file 4-24.wav. Silence removed file exists.\n",
            "(2/2): Processing file 2.mp3...\n",
            "\tMatching Start...\n",
            "\t\tPart [0]: Transcript= گزارش فعالیت‌های انجامن نابینایان ایران منصور شادکاه قوعمقام انجمن نابینایان ایران انجمن نابینایان ایران موسس خیری است که از هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی اجتماعی حمایتی رفاعی آموزشی\n",
            "\t\tMatched Text= از یک هزار و سیصد و نود و یک شروع شد؛ یعنی زمانی که اولین بار کتابخانه‌های ویژه نابینایان به همت شرکت سامسونگ پایه‌گذاری شد. تا یک هزار و سیصد و نود و هفت پانزده کتابخانه ویژه نابینایان در ایران تأسیس\n",
            "\t\tCER=0.6666666666666666\n",
            "\t\tPart [1]: Transcript= منصور شادکاه قوعمقام انجمن نابینایان ایران انجمن نابینایان ایران موسس خیری است که از هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی اجتماعی حمایتی رفاعی آموزشی\n",
            "\t\tMatched Text= منصور شادکام قائم‌مقام انجمن انجمن نابینایان ایران، مؤسسه‌ای خیریه است که از یک هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی، اجتماعی، حمایتی، رفاهی، آموزشی،\n",
            "\t\tCER=0.24\n",
            "\t\tPart [2]: Transcript= انجمن نابینایان ایران موسس خیری است که از هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی اجتماعی حمایتی رفاعی آموزشی مشاوره و توان بخشید تاسیس شده است\n",
            "\t\tMatched Text= نابینایان ایران، مؤسسه‌ای خیریه است که از یک هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی، اجتماعی، حمایتی، رفاهی، آموزشی، مشاوره و توانبخشی تأسیس‌ش\n",
            "\t\tCER=0.12666666666666668\n",
            "\t\tPart [3]: Transcript= که از هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی اجتماعی حمایتی رفاعی آموزشی مشاوره و توان بخشید تاسیس شده است این انجام من در تمام موضوعات فوق به جامعه هدف خدمات میرساند\n",
            "\t\tMatched Text= که از یک هزار و سیصد و هفتاد و سه با موضوع فعالیت فرهنگی، اجتماعی، حمایتی، رفاهی، آموزشی، مشاوره و توانبخشی تأسیس‌شده است. این انجمن در تمام موضوعات فوق به جامعه هدف خدمات می‌\n",
            "\t\tCER=0.0738255033557047\n",
            "\tMatching End...\n",
            "\t\tPart [0]: Transcript= با مشکلاتی نظیر ازدواج و مس کن حل می شود و آن‌ها از فرصت برابر باسایریم برخوردار خواهند شد در پایان از شرکت‌ها و ارگان‌هایی که ن‌هایت همکاری را با ما دارند به ویژه سامسانک تشکر و قدردانی میکنم\n",
            "\t\tMatched Text= یر ازدواج و مسکن حل می‌شود و آنها از فرصت برابر با سایرین برخوردار خواهند شد. در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، به ویژه سامسونگ، تشکر و قدردانی می‌کنم. فایل صوتی\n",
            "\t\tCER=0.16666666666666666\n",
            "\t\tPart [1]: Transcript= کرآوت انسانی نابینایان حفظ با مشکلاتی نظیر ازدواج و مس کن حل می شود و آن‌ها از فرصت برابر باسایریم برخوردار خواهند شد در پایان از شرکت‌ها و ارگان‌هایی که ن‌هایت همکاری را با ما دارند به ویژه سامسانک\n",
            "\t\tMatched Text= اتی نظیر ازدواج و مسکن حل می‌شود و آنها از فرصت برابر با سایرین برخوردار خواهند شد. در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، به ویژه سامسونگ، تشکر و قدردانی می‌کنم. فایل صوتی\n",
            "\t\tCER=0.44666666666666666\n",
            "\t\tPart [2]: Transcript= با رفع مشکل اجتبال کرآوت انسانی نابینایان حفظ با مشکلاتی نظیر ازدواج و مس کن حل می شود و آن‌ها از فرصت برابر باسایریم برخوردار خواهند شد در پایان از شرکت‌ها و ارگان‌هایی که ن‌هایت همکاری را با ما دارند\n",
            "\t\tMatched Text= شکلاتی نظیر ازدواج و مسکن حل می‌شود و آنها از فرصت برابر با سایرین برخوردار خواهند شد. در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، به ویژه سامسونگ، تشکر و قدردانی می‌کنم. فایل صوتی\n",
            "\t\tCER=0.66\n",
            "\t\tPart [3]: Transcript= و بتوانیم شغل دائمی را برای نابینایان دست و پا کنیم با رفع مشکل اجتبال کرآوت انسانی نابینایان حفظ با مشکلاتی نظیر ازدواج و مس کن حل می شود و آن‌ها از فرصت برابر باسایریم برخوردار خواهند شد در پایان\n",
            "\t\tMatched Text= تی نظیر ازدواج و مسکن حل می‌شود و آنها از فرصت برابر با سایرین برخوردار خواهند شد. در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، به ویژه سامسونگ، تشکر و قدردانی می‌کنم. فایل صوتی\n",
            "\t\tCER=0.7666666666666667\n",
            "\t\tPart [4]: Transcript= و بتوانیم شغل دائمی را برای نابینایان دست و پا کنیم با رفع مشکل اجتبال کرآوت انسانی نابینایان حفظ با مشکلاتی نظیر ازدواج و مس کن حل می شود و آن‌ها از فرصت برابر باسایریم برخوردار خواهند شد\n",
            "\t\tMatched Text= ا سایرین برخوردار خواهند شد. در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، به ویژه سامسونگ، تشکر و قدردانی می‌کنم. فایل صوتی صحبت‌های قائم‌مقام انجمن را از همینجا بشنوید\n",
            "\t\tCER=0.7666666666666667\n",
            "Matching chunk 1/138 of 2.wav...\n",
            "\tTranscript:\t\t انجمن نابینایان ایران موسس خیری است\n",
            "\tInterval Search: Matching Text: نابینایان ایران، مؤسسه‌ای خیریه است \n",
            "\tInterval Search: CER: 0.35294117647058826, End Pointer: 36\n",
            "\u001b[91m\tBoth Searches: Matching Text:نابینایان ایران، مؤسسه‌ای خیریه است \u001b[0m\n",
            "\u001b[91m\tBoth Searches: CER: 0.35294117647058826, End Pointer: 36\u001b[0m\n",
            "\tTranscript:\t\t انجمن نابینایان ایران موسس خی\n",
            "\tInterval Search: Matching Text: نابینایان ایران، مؤسسه‌ای \n",
            "\tInterval Search: CER: 0.375, End Pointer: 26\n",
            "\u001b[91m\tBoth Searches: Matching Text:نابینایان ایران، مؤسسه‌ای \u001b[0m\n",
            "\u001b[91m\tBoth Searches: CER: 0.375, End Pointer: 26\u001b[0m\n",
            "\tTranscript:\t\t انجامن نابینایان ایران موسسهای خیریه است\n",
            "\tInterval Search: Matching Text: نابینایان ایران، مؤسسه‌ای خیریه است \n",
            "\tInterval Search: CER: 0.2647058823529412, End Pointer: 36\n",
            "\u001b[91m\tBoth Searches: Matching Text:نابینایان ایران، مؤسسه‌ای خیریه است \u001b[0m\n",
            "\u001b[91m\tBoth Searches: CER: 0.2647058823529412, End Pointer: 36\u001b[0m\n",
            "\tWriting audio chunk 1...\n",
            "\tWriting text chunk 1...\n",
            "Matching chunk 2/138 of 2.wav...\n",
            "\tTranscript:\t\t که از هزار و سیصد و هفتاد و سه\n",
            "\u001b[93m\tInterval Search: Matching Text: که از یک هزار و سیصد و هفتاد و سه \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.09090909090909091, End Pointer: 70\u001b[0m\n",
            "\tWriting audio chunk 2...\n",
            "\tWriting text chunk 2...\n",
            "Matching chunk 3/138 of 2.wav...\n",
            "\tTranscript:\t\t با موضوع فعالیت فرهنگی اجتماعی حمایتی رفاعی آموزشی\n",
            "\u001b[92m\tInterval Search: Matching Text:با موضوع فعالیت فرهنگی، اجتماعی، حمایتی، رفاهی، آموزشی، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02, End Pointer: 71\u001b[0m\n",
            "\tWriting audio chunk 3...\n",
            "\tWriting text chunk 3...\n",
            "Matching chunk 4/138 of 2.wav...\n",
            "\tTranscript:\t\t مشاوره و توان بخشید تاسیس شده است\n",
            "\u001b[93m\tInterval Search: Matching Text: مشاوره و توانبخشی تأسیس‌شده است. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.0967741935483871, End Pointer: 48\u001b[0m\n",
            "\tWriting audio chunk 4...\n",
            "\tWriting text chunk 4...\n",
            "Matching chunk 5/138 of 2.wav...\n",
            "\tTranscript:\t\t این انجام من در تمام موضوعات فوق به جامعه هدف خدمات میرساند\n",
            "\u001b[93m\tInterval Search: Matching Text: این انجمن در تمام موضوعات فوق به جامعه هدف خدمات می‌رساند. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07017543859649122, End Pointer: 74\u001b[0m\n",
            "\tWriting audio chunk 5...\n",
            "\tWriting text chunk 5...\n",
            "Matching chunk 6/138 of 2.wav...\n",
            "\tTranscript:\t\t فعالیت‌های منستجم فرهنگی انجامن\n",
            "\u001b[93m\tInterval Search: Matching Text: فعالیت‌های منسجم فرهنگی انجمن \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06896551724137931, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 6...\n",
            "\tWriting text chunk 6...\n",
            "Matching chunk 7/138 of 2.wav...\n",
            "\tTranscript:\t\t در واقع از هزار و سیصد و نود و یک شروع شد\n",
            "\u001b[93m\tInterval Search: Matching Text: در واقع از یک هزار و سیصد و نود و یک شروع شد؛ \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06818181818181818, End Pointer: 61\u001b[0m\n",
            "\tWriting audio chunk 7...\n",
            "\tWriting text chunk 7...\n",
            "Matching chunk 8/138 of 2.wav...\n",
            "\tTranscript:\t\t یعنی زمانی که اولین بار کتابخانه‌های ویژه نابینایان\n",
            "\u001b[92m\tInterval Search: Matching Text:یعنی زمانی که اولین بار کتابخانه‌های ویژه نابینایان \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 8...\n",
            "\tWriting text chunk 8...\n",
            "Matching chunk 9/138 of 2.wav...\n",
            "\tTranscript:\t\t به همت شرکت سامسون پایه گذاری شد\n",
            "\u001b[92m\tInterval Search: Matching Text:به همت شرکت سامسونگ پایه‌گذاری شد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.030303030303030304, End Pointer: 50\u001b[0m\n",
            "\tWriting audio chunk 9...\n",
            "\tWriting text chunk 9...\n",
            "Matching chunk 10/138 of 2.wav...\n",
            "\tTranscript:\t\t تا هزار و سیصد و نود و هفت\n",
            "\u001b[93m\tInterval Search: Matching Text: یک هزار و سیصد و نود و هفت \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07692307692307693, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 10...\n",
            "\tWriting text chunk 10...\n",
            "Matching chunk 11/138 of 2.wav...\n",
            "\tTranscript:\t\t پانزده کتابخانه ویژه نابینایان در ایران تاسیل شد\n",
            "\u001b[92m\tInterval Search: Matching Text:پانزده کتابخانه ویژه نابینایان در ایران تأسیس شد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.041666666666666664, End Pointer: 65\u001b[0m\n",
            "\tWriting audio chunk 11...\n",
            "\tWriting text chunk 11...\n",
            "Matching chunk 12/138 of 2.wav...\n",
            "\tTranscript:\t\t در این کتابخانه‌ها تمام تجهیزاته بهروز افراد نابینا موجود است\n",
            "\u001b[92m\tInterval Search: Matching Text:در این کتابخانه‌ها تمام تجهیزات به‌روز افراد نابینا موجود است \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03278688524590164, End Pointer: 77\u001b[0m\n",
            "\tWriting audio chunk 12...\n",
            "\tWriting text chunk 12...\n",
            "Matching chunk 13/138 of 2.wav...\n",
            "\tTranscript:\t\t و کاربران مراجهان نابینا میتواند\n",
            "\u001b[93m\tInterval Search: Matching Text: و کاربران و مراجعان نابینا می‌توانند \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1388888888888889, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 13...\n",
            "\tWriting text chunk 13...\n",
            "Matching chunk 14/138 of 2.wav...\n",
            "\tTranscript:\t\t به مدل این تجهیزات نیاز‌های مطالعاتی خود را بر ترف کند\n",
            "\u001b[93m\tInterval Search: Matching Text: به مدد این تجهیزات، نیاز‌های مطالعاتی خود را برطرف کنند. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07407407407407407, End Pointer: 72\u001b[0m\n",
            "\tWriting audio chunk 14...\n",
            "\tWriting text chunk 14...\n",
            "Matching chunk 15/138 of 2.wav...\n",
            "\tTranscript:\t\t غیر از ایجاد این کتابخانه‌ها\n",
            "\u001b[92m\tInterval Search: Matching Text:غیر از ایجاد این کتابخانه‌ها، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 15...\n",
            "\tWriting text chunk 15...\n",
            "Matching chunk 16/138 of 2.wav...\n",
            "\tTranscript:\t\t انجمن در طول این چند سال چند جشن واره فرهنگی برگزار کرده است\n",
            "\u001b[92m\tInterval Search: Matching Text:انجمن در طول این چند سال چند جشنواره فرهنگی برگزار کرده است. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.01694915254237288, End Pointer: 76\u001b[0m\n",
            "\tWriting audio chunk 16...\n",
            "\tWriting text chunk 16...\n",
            "Matching chunk 17/138 of 2.wav...\n",
            "\tTranscript:\t\t از آن جمله میتوانم ب برگزاری اولین دوره مسابقات قرعان نابینایان را کمبینایان در سطح کشور\n",
            "\u001b[93m\tInterval Search: Matching Text: از آن جمله می‌توانم به برگزاری اولین دوره مسابقات قرآن نابینایان و کم بینایان در سطح کشور \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07865168539325842, End Pointer: 105\u001b[0m\n",
            "\tWriting audio chunk 17...\n",
            "\tWriting text chunk 17...\n",
            "Matching chunk 18/138 of 2.wav...\n",
            "\tTranscript:\t\t در هزار و سیصد و نود و سه اشاره کنم\n",
            "\u001b[93m\tInterval Search: Matching Text: یک هزار و سیصد و نود و سه اشاره کنم. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05714285714285714, End Pointer: 55\u001b[0m\n",
            "\tWriting audio chunk 18...\n",
            "\tWriting text chunk 18...\n",
            "Matching chunk 19/138 of 2.wav...\n",
            "\tTranscript:\t\t اولین جشنواره کتاب نابینایی و کمبینایی در هزار و سیصد و نود و شش\n",
            "\u001b[93m\tInterval Search: Matching Text: اولین جشنواره کتاب نابینایی و کم بینایی در یک هزار و سیصد و نود و شش \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.058823529411764705, End Pointer: 84\u001b[0m\n",
            "\tWriting audio chunk 19...\n",
            "\tWriting text chunk 19...\n",
            "Matching chunk 20/138 of 2.wav...\n",
            "\tTranscript:\t\t و برگزاری جشنواره کاما\n",
            "\u001b[92m\tInterval Search: Matching Text:و برگزاری جشنواره کاما \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 38\u001b[0m\n",
            "\tWriting audio chunk 20...\n",
            "\tWriting text chunk 20...\n",
            "Matching chunk 21/138 of 2.wav...\n",
            "\tTranscript:\t\t کتاب افراد دارای معلولیت ایران\n",
            "\u001b[92m\tInterval Search: Matching Text:کتاب افراد دارای معلولیت ایران \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 21...\n",
            "\tWriting text chunk 21...\n",
            "Matching chunk 22/138 of 2.wav...\n",
            "\tTranscript:\t\t در هزار و سیصد و نود و هشت\n",
            "\u001b[93m\tInterval Search: Matching Text: یک هزار و سیصد و نود و هشت \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07692307692307693, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 22...\n",
            "\tWriting text chunk 22...\n",
            "Matching chunk 23/138 of 2.wav...\n",
            "\tTranscript:\t\t از دیگر فعالیات‌های فرهنگی بود که در طول یک دهه گذشته\n",
            "\u001b[92m\tInterval Search: Matching Text:از دیگر فعالیت‌های فرهنگی بود که در طول یک دهه گذشته \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.019230769230769232, End Pointer: 68\u001b[0m\n",
            "\tWriting audio chunk 23...\n",
            "\tWriting text chunk 23...\n",
            "Matching chunk 24/138 of 2.wav...\n",
            "\tTranscript:\t\t به همت انجمن برگزار شده است\n",
            "\u001b[92m\tInterval Search: Matching Text:به همت انجمن برگزار شده است. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 44\u001b[0m\n",
            "\tWriting audio chunk 24...\n",
            "\tWriting text chunk 24...\n",
            "Matching chunk 25/138 of 2.wav...\n",
            "\tTranscript:\t\t اتفاقا این سرویداد بزرگ که هدف از آن\n",
            "\u001b[93m\tInterval Search: Matching Text: اتفاقا این سه رویداد بزرگ که هدف از آن \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05263157894736842, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 25...\n",
            "\tWriting text chunk 25...\n",
            "Matching chunk 26/138 of 2.wav...\n",
            "\tTranscript:\t\t شناساندن نخبگان و افراد توانمند جامعه نابینایان به جامعه\n",
            "\u001b[92m\tInterval Search: Matching Text:شناساندن نخبگان و افراد توانمند جامعه نابینایان به جامعه \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 72\u001b[0m\n",
            "\tWriting audio chunk 26...\n",
            "\tWriting text chunk 26...\n",
            "Matching chunk 27/138 of 2.wav...\n",
            "\tTranscript:\t\t و به خصوص مسئولین بود\n",
            "\u001b[92m\tInterval Search: Matching Text:و به خصوص مسؤولین بود، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.047619047619047616, End Pointer: 38\u001b[0m\n",
            "\tWriting audio chunk 27...\n",
            "\tWriting text chunk 27...\n",
            "Matching chunk 28/138 of 2.wav...\n",
            "\tTranscript:\t\t بسیار پرشور برگزار شد\n",
            "\u001b[92m\tInterval Search: Matching Text:بسیار پر‌شور برگزار شد \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.045454545454545456, End Pointer: 38\u001b[0m\n",
            "\tWriting audio chunk 28...\n",
            "\tWriting text chunk 28...\n",
            "Matching chunk 29/138 of 2.wav...\n",
            "\tTranscript:\t\t ما توانستیم به اهداف مدر نظرمان دست پیدا کنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:ما توانستیم به اهداف مد نظرمان دست پیدا کنیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.022727272727272728, End Pointer: 63\u001b[0m\n",
            "\tWriting audio chunk 29...\n",
            "\tWriting text chunk 29...\n",
            "Matching chunk 30/138 of 2.wav...\n",
            "\tTranscript:\t\t در حضر فرهنگی علاوه بر این\n",
            "\u001b[93m\tInterval Search: Matching Text: در حوزه فرهنگی علاوه بر این \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1111111111111111, End Pointer: 43\u001b[0m\n",
            "\tWriting audio chunk 30...\n",
            "\tWriting text chunk 30...\n",
            "Matching chunk 31/138 of 2.wav...\n",
            "\tTranscript:\t\t ما مسابقات هفته کتاب را برگزار کردیم\n",
            "\u001b[92m\tInterval Search: Matching Text:ما مسابقات هفته کتاب را برگزار کردیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 31...\n",
            "\tWriting text chunk 31...\n",
            "Matching chunk 32/138 of 2.wav...\n",
            "\tTranscript:\t\t که جوایز خوبی هم برای برگزیدگان این س زل برنامه‌ها در نظر گرفته شده بود\n",
            "\u001b[92m\tInterval Search: Matching Text:که جوایز خوبی هم برای برگزیدگان این سلسله برنامه‌ها در نظر گرفته‌شده بود. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.041666666666666664, End Pointer: 89\u001b[0m\n",
            "\tWriting audio chunk 32...\n",
            "\tWriting text chunk 32...\n",
            "Matching chunk 33/138 of 2.wav...\n",
            "\tTranscript:\t\t راه اندازه یک راهنامه الکترونیکی\n",
            "\u001b[93m\tInterval Search: Matching Text: راه‌اندازی یک ماهنامه الکترونیکی \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.0625, End Pointer: 48\u001b[0m\n",
            "\tWriting audio chunk 33...\n",
            "\tWriting text chunk 33...\n",
            "Matching chunk 34/138 of 2.wav...\n",
            "\tTranscript:\t\t شامل مطال ب صوتی و متنی\n",
            "\u001b[92m\tInterval Search: Matching Text:شامل مطالب صوتی و متنی، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.045454545454545456, End Pointer: 39\u001b[0m\n",
            "\tWriting audio chunk 34...\n",
            "\tWriting text chunk 34...\n",
            "Matching chunk 35/138 of 2.wav...\n",
            "\tTranscript:\t\t جدید‌ترین فعالیت انجمن در حوزه مسئولیت فرهنگی است\n",
            "\u001b[92m\tInterval Search: Matching Text:جدید‌ترین فعالیت انجمن در حوزه مسؤولیت فرهنگی است. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02040816326530612, End Pointer: 66\u001b[0m\n",
            "\tWriting audio chunk 35...\n",
            "\tWriting text chunk 35...\n",
            "Matching chunk 36/138 of 2.wav...\n",
            "\tTranscript:\t\t این مجله بالق برده سرویس دارد\n",
            "\u001b[93m\tInterval Search: Matching Text: این مجله، بالغ بر نه سرویس دارد \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1, End Pointer: 47\u001b[0m\n",
            "\tWriting audio chunk 36...\n",
            "\tWriting text chunk 36...\n",
            "Matching chunk 37/138 of 2.wav...\n",
            "\tTranscript:\t\t که همگی سرویس‌ها را افراد نابینای نخفه و صاحب قلم اداره میکنند\n",
            "\u001b[92m\tInterval Search: Matching Text:که همگی سرویس‌ها را افراد نابینای نخبه و صاحب‌قلم اداره می‌کنند. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.031746031746031744, End Pointer: 80\u001b[0m\n",
            "\tWriting audio chunk 37...\n",
            "\tWriting text chunk 37...\n",
            "Matching chunk 38/138 of 2.wav...\n",
            "\tTranscript:\t\t افرادی که با زیر وبم جامعه نابینایان به حد کافی آشنا هستند\n",
            "\u001b[92m\tInterval Search: Matching Text:افرادی که با زیر و بم جامعه نابینایان به حد کافی آشنا هستند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.01694915254237288, End Pointer: 75\u001b[0m\n",
            "\tWriting audio chunk 38...\n",
            "\tWriting text chunk 38...\n",
            "Matching chunk 39/138 of 2.wav...\n",
            "\tTranscript:\t\t و نیاز جامعه مخاطب را به خوبی میشناسند\n",
            "\u001b[92m\tInterval Search: Matching Text:و نیاز جامعه مخاطب را به خوبی می‌شناسند. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02564102564102564, End Pointer: 56\u001b[0m\n",
            "\tWriting audio chunk 39...\n",
            "\tWriting text chunk 39...\n",
            "Matching chunk 40/138 of 2.wav...\n",
            "\tTranscript:\t\t هدف انجمن این است که بتواند ضمن افزایش تعداد سرویس‌های ماهنامه\n",
            "\u001b[92m\tInterval Search: Matching Text:هدف انجمن این است که بتواند ضمن افزایش تعداد سرویس‌های ماهنامه، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 79\u001b[0m\n",
            "\tWriting audio chunk 40...\n",
            "\tWriting text chunk 40...\n",
            "Matching chunk 41/138 of 2.wav...\n",
            "\tTranscript:\t\t این مجله الکترونیکی را\n",
            "\u001b[92m\tInterval Search: Matching Text:این مجله الکترونیکی را \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 38\u001b[0m\n",
            "\tWriting audio chunk 41...\n",
            "\tWriting text chunk 41...\n",
            "Matching chunk 42/138 of 2.wav...\n",
            "\tTranscript:\t\t به شکل کاغذی و به خط بریل هم منتشر کند\n",
            "\u001b[92m\tInterval Search: Matching Text:به شکل کاغذی و به خط بریل هم منتشر کند. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 55\u001b[0m\n",
            "\tWriting audio chunk 42...\n",
            "\tWriting text chunk 42...\n",
            "Matching chunk 43/138 of 2.wav...\n",
            "\tTranscript:\t\t لازم است حتما به این نکته اشاره شود که بانک مس کر\n",
            "\u001b[92m\tInterval Search: Matching Text:لازم است حتما به این نکته اشاره شود که بانک مسکن، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.041666666666666664, End Pointer: 65\u001b[0m\n",
            "\tWriting audio chunk 43...\n",
            "\tWriting text chunk 43...\n",
            "Matching chunk 44/138 of 2.wav...\n",
            "\tTranscript:\t\t با حمایت مالی خود به ما کمک کرده که بتوانیم\n",
            "\u001b[92m\tInterval Search: Matching Text:با حمایت مالی خود به ما کمک کرده که بتوانیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 59\u001b[0m\n",
            "\tWriting audio chunk 44...\n",
            "\tWriting text chunk 44...\n",
            "Matching chunk 45/138 of 2.wav...\n",
            "\tTranscript:\t\t این نشریه را راه اندازی کنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:این نشریه را راه‌اندازی کنیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 45...\n",
            "\tWriting text chunk 45...\n",
            "Matching chunk 46/138 of 2.wav...\n",
            "\tTranscript:\t\t حضور حمایتی بانکم استکن در فعالیت‌های گوناگونه انجامن همیشه پر رنگ بوده\n",
            "\u001b[93m\tInterval Search: Matching Text: حضور حمایتی بانک مسکن در فعالیت‌های گوناگون انجمن همیشه پر رنگ بوده \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07462686567164178, End Pointer: 83\u001b[0m\n",
            "\tWriting audio chunk 46...\n",
            "\tWriting text chunk 46...\n",
            "Matching chunk 47/138 of 2.wav...\n",
            "\tTranscript:\t\t و ما از این حضور تشکر میکنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و ما از این حضور تشکر می‌کنیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.034482758620689655, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 47...\n",
            "\tWriting text chunk 47...\n",
            "Matching chunk 48/138 of 2.wav...\n",
            "\tTranscript:\t\t و از نهاد‌ها سازمان‌ها شرکت‌ها و افراد حقیقی حقوقی دیگر هم میخواهیم که\n",
            "\u001b[92m\tInterval Search: Matching Text:و از نهاد‌ها، سازمان‌ها، شرکت‌ها و افراد حقیقی و حقوقی دیگر هم می‌خواهیم که \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0410958904109589, End Pointer: 91\u001b[0m\n",
            "\tWriting audio chunk 48...\n",
            "\tWriting text chunk 48...\n",
            "Matching chunk 49/138 of 2.wav...\n",
            "\tTranscript:\t\t یا رویاور ما در این زمینه باشد\n",
            "\u001b[93m\tInterval Search: Matching Text: یار و یاور ما در این زمینه باشند \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.125, End Pointer: 48\u001b[0m\n",
            "\tWriting audio chunk 49...\n",
            "\tWriting text chunk 49...\n",
            "Matching chunk 50/138 of 2.wav...\n",
            "\tTranscript:\t\t و با حمایتشان از انجمن نابینایان ایران\n",
            "\u001b[92m\tInterval Search: Matching Text:و با حمایتشان از انجمن نابینایان ایران \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 50...\n",
            "\tWriting text chunk 50...\n",
            "Matching chunk 51/138 of 2.wav...\n",
            "\tTranscript:\t\t کمک کنند بتوانیم خدمات مصلوب‌تری را\n",
            "\u001b[92m\tInterval Search: Matching Text:کمک کنند بتوانیم خدمات مطلوب‌تری را \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02857142857142857, End Pointer: 51\u001b[0m\n",
            "\tWriting audio chunk 51...\n",
            "\tWriting text chunk 51...\n",
            "Matching chunk 52/138 of 2.wav...\n",
            "\tTranscript:\t\t در اختیار جامعه افراد آسیب دیده بینایی قرار دهیم\n",
            "\u001b[92m\tInterval Search: Matching Text:در اختیار جامعه افراد آسیب‌دیده بینایی قرار دهیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 65\u001b[0m\n",
            "\tWriting audio chunk 52...\n",
            "\tWriting text chunk 52...\n",
            "Matching chunk 53/138 of 2.wav...\n",
            "\tTranscript:\t\t در زمینه حمایت از افشار آسیب پذیر نابینا و کمبینال\n",
            "\u001b[93m\tInterval Search: Matching Text: در زمینه حمایت از اقشار آسیب‌پذیر نابینا و کم‌بینا، \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 53...\n",
            "\tWriting text chunk 53...\n",
            "Matching chunk 54/138 of 2.wav...\n",
            "\tTranscript:\t\t انجمن هر سال چهار نوبت توزیع ثبد کالا\n",
            "\u001b[92m\tInterval Search: Matching Text:انجمن هر سال چهار نوبت توزیع سبد کالا \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02702702702702703, End Pointer: 53\u001b[0m\n",
            "\tWriting audio chunk 54...\n",
            "\tWriting text chunk 54...\n",
            "Matching chunk 55/138 of 2.wav...\n",
            "\tTranscript:\t\t شامل ارزاغ مورد نیاز این خانواده‌ها را در دستور کار دارد\n",
            "\u001b[92m\tInterval Search: Matching Text:شامل ارزاق مورد نیاز این خانواده‌ها را در دستور کار دارد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.017857142857142856, End Pointer: 73\u001b[0m\n",
            "\tWriting audio chunk 55...\n",
            "\tWriting text chunk 55...\n",
            "Matching chunk 56/138 of 2.wav...\n",
            "\tTranscript:\t\t کمک‌های در زمینه تهیه جهیزیه برای زوج‌های جوان\n",
            "\u001b[92m\tInterval Search: Matching Text:کمک‌هایی در زمینه تهیه جهیزیه برای زوج‌های جوان \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02127659574468085, End Pointer: 63\u001b[0m\n",
            "\tWriting audio chunk 56...\n",
            "\tWriting text chunk 56...\n",
            "Matching chunk 57/138 of 2.wav...\n",
            "\tTranscript:\t\t و مجظومات زندگی برای خانواده‌های ناوینا\n",
            "\u001b[93m\tInterval Search: Matching Text: و ملزومات زندگی برای خانواده‌های نابینا، \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07692307692307693, End Pointer: 56\u001b[0m\n",
            "\tWriting audio chunk 57...\n",
            "\tWriting text chunk 57...\n",
            "Matching chunk 58/138 of 2.wav...\n",
            "\tTranscript:\t\t از دیگر فعالیت‌های حمایتی انجامم محسوب میشود\n",
            "\u001b[93m\tInterval Search: Matching Text: از دیگر فعالیت‌های حمایتی انجمن محسوب می‌شود. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06818181818181818, End Pointer: 61\u001b[0m\n",
            "\tWriting audio chunk 58...\n",
            "\tWriting text chunk 58...\n",
            "Matching chunk 59/138 of 2.wav...\n",
            "\tTranscript:\t\t ما همچنین امسال به دلیل شیور و همهگیری ویروس کرونا\n",
            "\u001b[92m\tInterval Search: Matching Text:ما همچنین امسال به دلیل شیوع و همه‌گیری ویروس کرونا، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0392156862745098, End Pointer: 68\u001b[0m\n",
            "\tWriting audio chunk 59...\n",
            "\tWriting text chunk 59...\n",
            "Matching chunk 60/138 of 2.wav...\n",
            "\tTranscript:\t\t توزیع ماسک و دستکش\n",
            "\u001b[92m\tInterval Search: Matching Text:توزیع ماسک و دستکش \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 34\u001b[0m\n",
            "\tWriting audio chunk 60...\n",
            "\tWriting text chunk 60...\n",
            "Matching chunk 61/138 of 2.wav...\n",
            "\tTranscript:\t\t و ژل‌های ضدعفونی کننده را هم در میان نابینایان به عهده گرفتهایم\n",
            "\u001b[92m\tInterval Search: Matching Text:و ژل‌های ضد عفونی کننده را هم در میان نابینایان به عهده گرفته‌ایم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03076923076923077, End Pointer: 82\u001b[0m\n",
            "\tWriting audio chunk 61...\n",
            "\tWriting text chunk 61...\n",
            "Matching chunk 62/138 of 2.wav...\n",
            "\tTranscript:\t\t انجمن همچنین در راستای ارائه خدمات مشاوری\n",
            "\u001b[93m\tInterval Search: Matching Text: انجمن همچنین در راستای ارائه خدمات مشاوره‌ای، \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06818181818181818, End Pointer: 61\u001b[0m\n",
            "\tWriting audio chunk 62...\n",
            "\tWriting text chunk 62...\n",
            "Matching chunk 63/138 of 2.wav...\n",
            "\tTranscript:\t\t از روانشناسان و مددکاران مجرب\n",
            "\u001b[92m\tInterval Search: Matching Text:از روانشناسان و مددکاران مجرب \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 63...\n",
            "\tWriting text chunk 63...\n",
            "Matching chunk 64/138 of 2.wav...\n",
            "\tTranscript:\t\t برای بر طرف کردن این نیاز  ه اعضای خود هم بهره میگیرد\n",
            "\u001b[93m\tInterval Search: Matching Text: برای برطرف کردن این نیاز اعضای خود هم بهره می‌گیرد. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.08, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 64...\n",
            "\tWriting text chunk 64...\n",
            "Matching chunk 65/138 of 2.wav...\n",
            "\tTranscript:\t\t پیگیری یه مشکلات حوزه توام بخشید\n",
            "\tInterval Search: Matching Text: پیگیری مشکلات حوزه توانبخشی، \n",
            "\tInterval Search: CER: 0.2222222222222222, End Pointer: 44\n",
            "\u001b[91m\tBoth Searches: Matching Text:پیگیری مشکلات حوزه توانبخشی، \u001b[0m\n",
            "\u001b[91m\tBoth Searches: CER: 0.2222222222222222, End Pointer: 44\u001b[0m\n",
            "\tTranscript:\t\t پیگیری مشکلات حوزه توام بخشی\n",
            "\u001b[93m\tInterval Search: Matching Text: پیگیری مشکلات حوزه توانبخشی، \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07407407407407407, End Pointer: 44\u001b[0m\n",
            "\tWriting audio chunk 65...\n",
            "\tWriting text chunk 65...\n",
            "Matching chunk 66/138 of 2.wav...\n",
            "\tTranscript:\t\t از دیگر فعالیت‌هایی است که انجامن احتمام خاصی نسبت به ما میبرزد\n",
            "\u001b[93m\tInterval Search: Matching Text: از دیگر فعالیت‌هایی است که انجمن اهتمام خاصی نسبت به آن می‌ورزد. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.09523809523809523, End Pointer: 80\u001b[0m\n",
            "\tWriting audio chunk 66...\n",
            "\tWriting text chunk 66...\n",
            "Matching chunk 67/138 of 2.wav...\n",
            "\tTranscript:\t\t پیگیری مشکلات مربوط به پلاک ویژه افراد دارای معلولیت\n",
            "\u001b[92m\tInterval Search: Matching Text:پیگیری مشکلات مربوط به پلاک ویژه افراد دارای معلولیت، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 69\u001b[0m\n",
            "\tWriting audio chunk 67...\n",
            "\tWriting text chunk 67...\n",
            "Matching chunk 68/138 of 2.wav...\n",
            "\tTranscript:\t\t سهمیه سوخت و مواردی از این دست\n",
            "\u001b[92m\tInterval Search: Matching Text:سهمیه سوخت و مواردی از این دست \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 68...\n",
            "\tWriting text chunk 68...\n",
            "Matching chunk 69/138 of 2.wav...\n",
            "\tTranscript:\t\t هموارد در دستور کار انجام من بوده است\n",
            "\u001b[93m\tInterval Search: Matching Text: همواره در دستور کار انجمن بوده است. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.11764705882352941, End Pointer: 51\u001b[0m\n",
            "\tWriting audio chunk 69...\n",
            "\tWriting text chunk 69...\n",
            "Matching chunk 70/138 of 2.wav...\n",
            "\tTranscript:\t\t ما همچنین بطور مستمر نام نگاری‌ها و جلساتی را هم با سازمان بهزیستی و هیت محترم دولت داشتیم\n",
            "\u001b[93m\tInterval Search: Matching Text: ما همچنین به طور مستمر نامه‌نگاری‌ها و جلساتی را هم با سازمان بهزیستی و هیئت محترم دولت داشته‌ایم \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07216494845360824, End Pointer: 113\u001b[0m\n",
            "\tWriting audio chunk 70...\n",
            "\tWriting text chunk 70...\n",
            "Matching chunk 71/138 of 2.wav...\n",
            "\tTranscript:\t\t و بر حسل اجرای قانون جامعه حمایت از حقوق معلولان\n",
            "\u001b[92m\tInterval Search: Matching Text:و بر حسن اجرای قانون جامع حمایت از حقوق معلولان، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0425531914893617, End Pointer: 64\u001b[0m\n",
            "\tWriting audio chunk 71...\n",
            "\tWriting text chunk 71...\n",
            "Matching chunk 72/138 of 2.wav...\n",
            "\tTranscript:\t\t ترکید و نزارت کرده ایم\n",
            "\u001b[93m\tInterval Search: Matching Text: تأکید و نظارت کرده‌ایم. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.09090909090909091, End Pointer: 39\u001b[0m\n",
            "\tWriting audio chunk 72...\n",
            "\tWriting text chunk 72...\n",
            "Matching chunk 73/138 of 2.wav...\n",
            "\tTranscript:\t\t در همین راستا مواردی را مستقیما\n",
            "\u001b[92m\tInterval Search: Matching Text:در همین راستا مواردی را مستقیما \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 47\u001b[0m\n",
            "\tWriting audio chunk 73...\n",
            "\tWriting text chunk 73...\n",
            "Matching chunk 74/138 of 2.wav...\n",
            "\tTranscript:\t\t با دکتر جهانگیری معاون رئیس جمهور و علیخانی معاون ایشان مصرح کرده ایم\n",
            "\u001b[92m\tInterval Search: Matching Text:با دکتر جهانگیری معاون رئیس‌جمهوری و علیخانی، معاون ایشان مطرح کرده‌ایم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02857142857142857, End Pointer: 87\u001b[0m\n",
            "\tWriting audio chunk 74...\n",
            "\tWriting text chunk 74...\n",
            "Matching chunk 75/138 of 2.wav...\n",
            "\tTranscript:\t\t و همچنان آن را پیگیری میکنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و همچنان آن را پیگیری می‌کنیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.034482758620689655, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 75...\n",
            "\tWriting text chunk 75...\n",
            "Matching chunk 76/138 of 2.wav...\n",
            "\tTranscript:\t\t به هر حال وضعیت کنونی کشور در شرایط سختی قرار دارد\n",
            "\u001b[92m\tInterval Search: Matching Text:به هر حال وضعیت کنونی کشور در شرایط سختی قرار دارد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 76...\n",
            "\tWriting text chunk 76...\n",
            "Matching chunk 77/138 of 2.wav...\n",
            "\tTranscript:\t\t آسیبهای ناشی از همهگیری کرونا\n",
            "\u001b[93m\tInterval Search: Matching Text: آسیب‌های ناشی از همه‌گیری کرونا \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06451612903225806, End Pointer: 47\u001b[0m\n",
            "\tWriting audio chunk 77...\n",
            "\tWriting text chunk 77...\n",
            "Matching chunk 78/138 of 2.wav...\n",
            "\tTranscript:\t\t و فشار‌هایی که در پی تحریم‌های حد عکثری به کشور وارد شده\n",
            "\u001b[92m\tInterval Search: Matching Text:و فشار‌هایی که در پی تحریم‌های حد اکثری به کشور وارد شده \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.017857142857142856, End Pointer: 72\u001b[0m\n",
            "\tWriting audio chunk 78...\n",
            "\tWriting text chunk 78...\n",
            "Matching chunk 79/138 of 2.wav...\n",
            "\tTranscript:\t\t باعث می شود اجرای قانون جامعه\n",
            "\u001b[92m\tInterval Search: Matching Text:باعث می‌شود اجرای قانون جامع \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03571428571428571, End Pointer: 44\u001b[0m\n",
            "\tWriting audio chunk 79...\n",
            "\tWriting text chunk 79...\n",
            "Matching chunk 80/138 of 2.wav...\n",
            "\tTranscript:\t\t با چالش های متعددی رو به رو باشد\n",
            "\u001b[92m\tInterval Search: Matching Text:با چالش‌های متعددی رو به رو باشد \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 48\u001b[0m\n",
            "\tWriting audio chunk 80...\n",
            "\tWriting text chunk 80...\n",
            "Matching chunk 81/138 of 2.wav...\n",
            "\tTranscript:\t\t اما ما به هر حال به دنبال اجرای بیکم و کاست آن هستیم\n",
            "\u001b[92m\tInterval Search: Matching Text:اما ما به هر حال به دنبال اجرای بی کم و کاست آن هستیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.018867924528301886, End Pointer: 70\u001b[0m\n",
            "\tWriting audio chunk 81...\n",
            "\tWriting text chunk 81...\n",
            "Matching chunk 82/138 of 2.wav...\n",
            "\tTranscript:\t\t در راستای احقاق حقوق افراد نابینا و کمبینا\n",
            "\u001b[92m\tInterval Search: Matching Text:در راستای احقاق حقوق افراد نابینا و کم‌بینا \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.023255813953488372, End Pointer: 59\u001b[0m\n",
            "\tWriting audio chunk 82...\n",
            "\tWriting text chunk 82...\n",
            "Matching chunk 83/138 of 2.wav...\n",
            "\tTranscript:\t\t در حوزه توام بخشی امسال در نشسته که به مناسبت روز جهانی ایمنی اسای سفید\n",
            "\u001b[93m\tInterval Search: Matching Text: در حوزه توانبخشی، امسال در نشستی که به مناسبت روز جهانی ایمنی عصای سفید \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07142857142857142, End Pointer: 87\u001b[0m\n",
            "\tWriting audio chunk 83...\n",
            "\tWriting text chunk 83...\n",
            "Matching chunk 84/138 of 2.wav...\n",
            "\tTranscript:\t\t با حصور وزیر رفاهکار و امور اجتماعی\n",
            "\u001b[93m\tInterval Search: Matching Text: با حضور وزیر رفاه کار و امور اجتماعی \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05555555555555555, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 84...\n",
            "\tWriting text chunk 84...\n",
            "Matching chunk 85/138 of 2.wav...\n",
            "\tTranscript:\t\t و رییس سازمان بهزیستی در محل این سازمان برگزار شد\n",
            "\u001b[92m\tInterval Search: Matching Text:و رئیس سازمان بهزیستی در محل این سازمان برگزار شد، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02040816326530612, End Pointer: 66\u001b[0m\n",
            "\tWriting audio chunk 85...\n",
            "\tWriting text chunk 85...\n",
            "Matching chunk 86/138 of 2.wav...\n",
            "\tTranscript:\t\t مطالباتی را مطرح کردیم\n",
            "\u001b[92m\tInterval Search: Matching Text:مطالباتی را مطرح کردیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 38\u001b[0m\n",
            "\tWriting audio chunk 86...\n",
            "\tWriting text chunk 86...\n",
            "Matching chunk 87/138 of 2.wav...\n",
            "\tTranscript:\t\t کتاب حال به یکی از آن ها پاسخ قطع ایداده شده ازت\n",
            "\u001b[93m\tInterval Search: Matching Text: تا به حال به یکی از آنها پاسخ قطعی داده‌شده است. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1702127659574468, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 87...\n",
            "\tWriting text chunk 87...\n",
            "Matching chunk 88/138 of 2.wav...\n",
            "\tTranscript:\t\t در این نشست ما تقاضای خرید صدر دستگاه یادداشت بردار بریل را مطرح کردیم که در اختیار انجامن‌ها و کتابخانه‌های فعال در حوزه نابینای آن قرار گیرد\n",
            "\u001b[92m\tInterval Search: Matching Text:در این نشست ما تقاضای خرید صد دستگاه یادداشتبردار بریل را مطرح کردیم که در اختیار انجمن‌ها و کتابخانه‌های فعال در حوزه نابینایان قرار گیرد \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.036231884057971016, End Pointer: 154\u001b[0m\n",
            "\tWriting audio chunk 88...\n",
            "\tWriting text chunk 88...\n",
            "Matching chunk 89/138 of 2.wav...\n",
            "\tTranscript:\t\t که با پاسخ مثبت وزیر محترف مواجه شد\n",
            "\u001b[92m\tInterval Search: Matching Text:که با پاسخ مثبت وزیر محترم مواجه شد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02857142857142857, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 89...\n",
            "\tWriting text chunk 89...\n",
            "Matching chunk 90/138 of 2.wav...\n",
            "\tTranscript:\t\t ما همچنین ایراداتی هم به طرح مثبت زندگی مطرح کردیم\n",
            "\u001b[92m\tInterval Search: Matching Text:ما همچنین ایراداتی هم به طرح مثبت زندگی مطرح کردیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 67\u001b[0m\n",
            "\tWriting audio chunk 90...\n",
            "\tWriting text chunk 90...\n",
            "Matching chunk 91/138 of 2.wav...\n",
            "\tTranscript:\t\t در خصوص رفع مشکلات تردد نابینایان\n",
            "\u001b[92m\tInterval Search: Matching Text:در خصوص رفع مشکلات تردد نابینایان، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 50\u001b[0m\n",
            "\tWriting audio chunk 91...\n",
            "\tWriting text chunk 91...\n",
            "Matching chunk 92/138 of 2.wav...\n",
            "\tTranscript:\t\t جلسهای با سامانه حمل و نقل جان بازان و معلولان داشتیم\n",
            "\u001b[92m\tInterval Search: Matching Text:جلسه‌ای با سامانه حمل و نقل جانبازان و معلولان داشتیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03773584905660377, End Pointer: 69\u001b[0m\n",
            "\tWriting audio chunk 92...\n",
            "\tWriting text chunk 92...\n",
            "Matching chunk 93/138 of 2.wav...\n",
            "\tTranscript:\t\t و موضوعی کم بود اتومبیل ویژه ایابز‌هاب معلولین را مصرح کردیم\n",
            "\u001b[93m\tInterval Search: Matching Text: و موضوع کمبود اتومبیل ویژه ایاب و ذهاب معلولین را مطرح کردیم \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1, End Pointer: 76\u001b[0m\n",
            "\tWriting audio chunk 93...\n",
            "\tWriting text chunk 93...\n",
            "Matching chunk 94/138 of 2.wav...\n",
            "\tTranscript:\t\t که بنافقاشمی مووعنت حبل و نقل شهرداری تهران\n",
            "\u001b[93m\tInterval Search: Matching Text: که مناف هاشمی معاونت حمل و نقل شهرداری تهران \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1590909090909091, End Pointer: 60\u001b[0m\n",
            "\tWriting audio chunk 94...\n",
            "\tWriting text chunk 94...\n",
            "Matching chunk 95/138 of 2.wav...\n",
            "\tTranscript:\t\t قول‌هایی در این راستا به ما داده است\n",
            "\u001b[92m\tInterval Search: Matching Text:قول‌هایی در این راستا به ما داده است \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 95...\n",
            "\tWriting text chunk 95...\n",
            "Matching chunk 96/138 of 2.wav...\n",
            "\tTranscript:\t\t و قرار شده امسال و سال بعد تعداد این خودرو‌ها افزایش یابد\n",
            "\u001b[92m\tInterval Search: Matching Text:و قرار شده امسال و سال بعد تعداد این خودرو‌ها افزایش یابد \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 73\u001b[0m\n",
            "\tWriting audio chunk 96...\n",
            "\tWriting text chunk 96...\n",
            "Matching chunk 97/138 of 2.wav...\n",
            "\tTranscript:\t\t تا بخشی از مشکلات ترددی نابینایان و معلولین مرتفر شود\n",
            "\u001b[92m\tInterval Search: Matching Text:تا بخشی از مشکلات ترددی نابینایان و معلولان مرتفع شود. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03773584905660377, End Pointer: 70\u001b[0m\n",
            "\tWriting audio chunk 97...\n",
            "\tWriting text chunk 97...\n",
            "Matching chunk 98/138 of 2.wav...\n",
            "\tTranscript:\t\t خبر خوش دیگری که می توانیم به مخاطبان مان بدهیم\n",
            "\u001b[92m\tInterval Search: Matching Text:خبر خوش دیگری که می‌توانیم به مخاطبانمان بدهیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.021739130434782608, End Pointer: 62\u001b[0m\n",
            "\tWriting audio chunk 98...\n",
            "\tWriting text chunk 98...\n",
            "Matching chunk 99/138 of 2.wav...\n",
            "\tTranscript:\t\t این است که ما نزدیک به بیش ازم\n",
            "\u001b[92m\tInterval Search: Matching Text:این است که ما نزدیک به بیش از \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.034482758620689655, End Pointer: 45\u001b[0m\n",
            "\tWriting audio chunk 99...\n",
            "\tWriting text chunk 99...\n",
            "Matching chunk 100/138 of 2.wav...\n",
            "\tTranscript:\t\t دویست شوبه بانکی را به سامانه خدمات بانکی مجهز کردیم\n",
            "\u001b[93m\tInterval Search: Matching Text: دویست شعبه بانکی را به سامانه خدمات بانکی مجهز کرده‌ایم \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.07272727272727272, End Pointer: 71\u001b[0m\n",
            "\tWriting audio chunk 100...\n",
            "\tWriting text chunk 100...\n",
            "Matching chunk 101/138 of 2.wav...\n",
            "\tTranscript:\t\t وطیه جلسات و م کاتباتی که با بانک مرکزی و مسئولان مربوطه داشتیم\n",
            "\u001b[93m\tInterval Search: Matching Text: و طی جلسات و مکاتباتی که با بانک مرکزی و مسؤولان مربوطه داشته‌ایم \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1076923076923077, End Pointer: 81\u001b[0m\n",
            "\tWriting audio chunk 101...\n",
            "\tWriting text chunk 101...\n",
            "Matching chunk 102/138 of 2.wav...\n",
            "\tTranscript:\t\t توانستیم قول بگیریم که مراکز بیشتری به این سامان مجهز شوند\n",
            "\u001b[92m\tInterval Search: Matching Text:توانستیم قول بگیریم که مراکز بیشتری به این سامانه مجهز شوند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.01694915254237288, End Pointer: 75\u001b[0m\n",
            "\tWriting audio chunk 102...\n",
            "\tWriting text chunk 102...\n",
            "Matching chunk 103/138 of 2.wav...\n",
            "\tTranscript:\t\t کنابینایان بتوانند در ن‌هایت استقلال از خدمات بانکی استفاده کنند\n",
            "\u001b[92m\tInterval Search: Matching Text:که نابینایان بتوانند در نهایت استقلال از خدمات بانکی استفاده کنند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.046153846153846156, End Pointer: 81\u001b[0m\n",
            "\tWriting audio chunk 103...\n",
            "\tWriting text chunk 103...\n",
            "Matching chunk 104/138 of 2.wav...\n",
            "\tTranscript:\t\t و کرآمت انسانی آن‌ها حف شود\n",
            "\u001b[93m\tInterval Search: Matching Text: و کرامت انسانی آنها حفظ شود. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1111111111111111, End Pointer: 44\u001b[0m\n",
            "\tWriting audio chunk 104...\n",
            "\tWriting text chunk 104...\n",
            "Matching chunk 105/138 of 2.wav...\n",
            "\tTranscript:\t\t مشکلات اجتقال حقیقتا مشکل اصلی نابینایان است\n",
            "\u001b[93m\tInterval Search: Matching Text: مشکل اشتغال، حقیقتا مشکل اصلی نابینایان است \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.09523809523809523, End Pointer: 59\u001b[0m\n",
            "\tWriting audio chunk 105...\n",
            "\tWriting text chunk 105...\n",
            "Matching chunk 106/138 of 2.wav...\n",
            "\tTranscript:\t\t و ما هم در این راستا به جدپیگیر هستیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و ما هم در این راستا به جد پیگیر هستیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02631578947368421, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 106...\n",
            "\tWriting text chunk 106...\n",
            "Matching chunk 107/138 of 2.wav...\n",
            "\tTranscript:\t\t و نام نگاری‌های زیادی در این راستا داشته ایم\n",
            "\u001b[92m\tInterval Search: Matching Text:و نامه‌نگاری‌های زیادی در این راستا داشته‌ایم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.022222222222222223, End Pointer: 62\u001b[0m\n",
            "\tWriting audio chunk 107...\n",
            "\tWriting text chunk 107...\n",
            "Matching chunk 108/138 of 2.wav...\n",
            "\tTranscript:\t\t یکی از فعالیت‌های همیشگی انجامن این بوده که\n",
            "\u001b[92m\tInterval Search: Matching Text:یکی از فعالیت‌های همیشگی انجمن این بوده که \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.023809523809523808, End Pointer: 58\u001b[0m\n",
            "\tWriting audio chunk 108...\n",
            "\tWriting text chunk 108...\n",
            "Matching chunk 109/138 of 2.wav...\n",
            "\tTranscript:\t\t در موقعیت‌های شغلی که نیاز به نیرو دارند\n",
            "\u001b[92m\tInterval Search: Matching Text:در موقعیت‌های شغلی که نیاز به نیرو دارند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 56\u001b[0m\n",
            "\tWriting audio chunk 109...\n",
            "\tWriting text chunk 109...\n",
            "Matching chunk 110/138 of 2.wav...\n",
            "\tTranscript:\t\t ما سعی کرده ایم کارفرمایان را متقاعد کنیم که قانون س درصد را روایت کند\n",
            "\u001b[92m\tInterval Search: Matching Text:ما سعی کرده‌ایم کارفرمایان را متقاعد کنیم که قانون سه درصد را رعایت کنند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.041666666666666664, End Pointer: 88\u001b[0m\n",
            "\tWriting audio chunk 110...\n",
            "\tWriting text chunk 110...\n",
            "Matching chunk 111/138 of 2.wav...\n",
            "\tTranscript:\t\t و از نیروی کار و نابینه‌ها بهره بگیرد\n",
            "\u001b[93m\tInterval Search: Matching Text: و از نیروی کار نابینا بهره بگیرند. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.18181818181818182, End Pointer: 50\u001b[0m\n",
            "\tWriting audio chunk 111...\n",
            "\tWriting text chunk 111...\n",
            "Matching chunk 112/138 of 2.wav...\n",
            "\tTranscript:\t\t به هر حال ما باید مغوله مسئولیت اجتماعی را جدی بگیریم\n",
            "\u001b[92m\tInterval Search: Matching Text:به هر حال ما باید مقوله مسؤولیت اجتماعی را جدی بگیریم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03773584905660377, End Pointer: 69\u001b[0m\n",
            "\tWriting audio chunk 112...\n",
            "\tWriting text chunk 112...\n",
            "Matching chunk 113/138 of 2.wav...\n",
            "\tTranscript:\t\t و تلاش کنیم آن را به عنوان یک طرز فکر در جامعه جا بیاندازیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و تلاش کنیم آن را به عنوان یک طرز فکر در جامعه جا بیندازیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.017241379310344827, End Pointer: 75\u001b[0m\n",
            "\tWriting audio chunk 113...\n",
            "\tWriting text chunk 113...\n",
            "Matching chunk 114/138 of 2.wav...\n",
            "\tTranscript:\t\t شرکت‌ها و ارگان‌ها باید به وظیفه انسانی و اخلاویه خود عمل کنند\n",
            "\u001b[92m\tInterval Search: Matching Text:شرکت‌ها و ارگان‌ها باید به وظیفه انسانی و اخلاقی خود عمل کنند. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.03278688524590164, End Pointer: 78\u001b[0m\n",
            "\tWriting audio chunk 114...\n",
            "\tWriting text chunk 114...\n",
            "Matching chunk 115/138 of 2.wav...\n",
            "\tTranscript:\t\t اگر تمام نهاد‌های خصوصی و دولتی پای کار بیایند\n",
            "\u001b[92m\tInterval Search: Matching Text:اگر تمام نهاد‌های خصوصی و دولتی پای کار بیایند، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 63\u001b[0m\n",
            "\tWriting audio chunk 115...\n",
            "\tWriting text chunk 115...\n",
            "Matching chunk 116/138 of 2.wav...\n",
            "\tTranscript:\t\t هرکدام حتی یک نفر نیروی نابینا و کمینا را مشغول بکار کند قطعا مشکل اشتقال نابینایان حل خواهد شد\n",
            "\u001b[93m\tInterval Search: Matching Text: هر کدام حتی یک نفر نیروی نابینا و کم‌بینا را مشغول به کار کنند قطعا مشکل اشتغال نابینایان حل خواهد شد. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06930693069306931, End Pointer: 118\u001b[0m\n",
            "\tWriting audio chunk 116...\n",
            "\tWriting text chunk 116...\n",
            "Matching chunk 117/138 of 2.wav...\n",
            "\tTranscript:\t\t نابینایان افراد توانمندی هستند\n",
            "\u001b[92m\tInterval Search: Matching Text:نابینایان افراد توانمندی هستند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 117...\n",
            "\tWriting text chunk 117...\n",
            "Matching chunk 118/138 of 2.wav...\n",
            "\tTranscript:\t\t که اگر امکانات در اختیارشان قرار گیرد\n",
            "\u001b[92m\tInterval Search: Matching Text:که اگر امکانات در اختیارشان قرار گیرد، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 118...\n",
            "\tWriting text chunk 118...\n",
            "Matching chunk 119/138 of 2.wav...\n",
            "\tTranscript:\t\t مطمئنا در محیط‌های شغلی چیزی از نیروی کار بینا کم نخواهند داشت\n",
            "\u001b[92m\tInterval Search: Matching Text:مطمئنا در محیط‌های شغلی چیزی از نیروی کار بینا کم نخواهند داشت. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 79\u001b[0m\n",
            "\tWriting audio chunk 119...\n",
            "\tWriting text chunk 119...\n",
            "Matching chunk 120/138 of 2.wav...\n",
            "\tTranscript:\t\t قطعا اینکه بگوییم معسط و کیفیت خدماتی که ارائه میکنیم رضایت کامل داریم سخن گذاف است\n",
            "\u001b[93m\tInterval Search: Matching Text: قطعا اینکه بگوییم ما از سطح و کیفیت خدماتی که ارائه می‌کنیم رضایت کامل داریم، سخنی گزاف است. \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.1, End Pointer: 108\u001b[0m\n",
            "\tWriting audio chunk 120...\n",
            "\tWriting text chunk 120...\n",
            "Matching chunk 121/138 of 2.wav...\n",
            "\tTranscript:\t\t شاید توانسته باشیم بیست و پنج درصد\n",
            "\u001b[92m\tInterval Search: Matching Text:شاید توانسته باشیم بیست و پنج درصد \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 50\u001b[0m\n",
            "\tWriting audio chunk 121...\n",
            "\tWriting text chunk 121...\n",
            "Matching chunk 122/138 of 2.wav...\n",
            "\tTranscript:\t\t به وظایف خودمان عمل کرده باشیم\n",
            "\u001b[92m\tInterval Search: Matching Text:به وظایف خودمان عمل کرده باشیم \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 122...\n",
            "\tWriting text chunk 122...\n",
            "Matching chunk 123/138 of 2.wav...\n",
            "\tTranscript:\t\t و به مشکلات نابینایان رسیدگی کنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و به مشکلات نابینایان رسیدگی کنیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 50\u001b[0m\n",
            "\tWriting audio chunk 123...\n",
            "\tWriting text chunk 123...\n",
            "Matching chunk 124/138 of 2.wav...\n",
            "\tTranscript:\t\t مشکلات و معضلات بسیار زیادی هستند\n",
            "\u001b[93m\tInterval Search: Matching Text: مشکلات و معضلات بسیار زیاد است \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.13333333333333333, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 124...\n",
            "\tWriting text chunk 124...\n",
            "Matching chunk 125/138 of 2.wav...\n",
            "\tTranscript:\t\t اما متاسفانه با این تعداد نی روی اندک\n",
            "\u001b[93m\tInterval Search: Matching Text: اما متأسفانه با این تعداد نیروی اندک \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05555555555555555, End Pointer: 52\u001b[0m\n",
            "\tWriting audio chunk 125...\n",
            "\tWriting text chunk 125...\n",
            "Matching chunk 126/138 of 2.wav...\n",
            "\tTranscript:\t\t و زیرساخت‌های محدودی که در اختیار داریم\n",
            "\u001b[92m\tInterval Search: Matching Text:و زیرساخت‌های محدودی که در اختیار داریم، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 56\u001b[0m\n",
            "\tWriting audio chunk 126...\n",
            "\tWriting text chunk 126...\n",
            "Matching chunk 127/138 of 2.wav...\n",
            "\tTranscript:\t\t قادر به پاسخگویی همه مراج آن نیستیم\n",
            "\u001b[93m\tInterval Search: Matching Text: قادر به پاسخگویی همه مراجعان نیستیم \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05714285714285714, End Pointer: 51\u001b[0m\n",
            "\tWriting audio chunk 127...\n",
            "\tWriting text chunk 127...\n",
            "Matching chunk 128/138 of 2.wav...\n",
            "\tTranscript:\t\t و شرمنده آن‌ها هستیم انشااله دوستان نابینا هم به ما کمک کنند\n",
            "\u001b[92m\tInterval Search: Matching Text:و شرمنده آنها هستیم. انشا الله دوستان نابینا هم به ما کمک کنند \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.04918032786885246, End Pointer: 78\u001b[0m\n",
            "\tWriting audio chunk 128...\n",
            "\tWriting text chunk 128...\n",
            "Matching chunk 129/138 of 2.wav...\n",
            "\tTranscript:\t\t و همراه ما باشند تا با همکاری اساسی و اصولی را صورت دهیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و همراه ما باشند تا با هم کاری اساسی و اصولی را صورت دهیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.017543859649122806, End Pointer: 74\u001b[0m\n",
            "\tWriting audio chunk 129...\n",
            "\tWriting text chunk 129...\n",
            "Matching chunk 130/138 of 2.wav...\n",
            "\tTranscript:\t\t امید ما این است که کار ما کاری اساسی باشد\n",
            "\u001b[92m\tInterval Search: Matching Text:امید ما این است که کار ما کاری اساسی باشد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 58\u001b[0m\n",
            "\tWriting audio chunk 130...\n",
            "\tWriting text chunk 130...\n",
            "Matching chunk 131/138 of 2.wav...\n",
            "\tTranscript:\t\t هدف اصلی و ن‌هائی ما این است که\n",
            "\u001b[93m\tInterval Search: Matching Text: هدف اصلی و نهایی ما این است که \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06666666666666667, End Pointer: 46\u001b[0m\n",
            "\tWriting audio chunk 131...\n",
            "\tWriting text chunk 131...\n",
            "Matching chunk 132/138 of 2.wav...\n",
            "\tTranscript:\t\t در زمینه اشتقال کاریماندگار صورت گیرد\n",
            "\u001b[93m\tInterval Search: Matching Text: در زمینه اشتغال کاری ماندگار صورت گیرد \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05263157894736842, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 132...\n",
            "\tWriting text chunk 132...\n",
            "Matching chunk 133/138 of 2.wav...\n",
            "\tTranscript:\t\t و بتوانیم شغل دائمی را برای نابینایان دست و پا کنیم\n",
            "\u001b[92m\tInterval Search: Matching Text:و بتوانیم شغل دائمی را برای نابینایان دست و پا کنیم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.0, End Pointer: 68\u001b[0m\n",
            "\tWriting audio chunk 133...\n",
            "\tWriting text chunk 133...\n",
            "Matching chunk 134/138 of 2.wav...\n",
            "\tTranscript:\t\t با رفع مشکل اجتبال کرآمت انسانی نابینایان حفظ\n",
            "\u001b[93m\tInterval Search: Matching Text: با رفع مشکل اشتغال، کرامت انسانی نابینایان حفظ \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.06666666666666667, End Pointer: 62\u001b[0m\n",
            "\tWriting audio chunk 134...\n",
            "\tWriting text chunk 134...\n",
            "Matching chunk 135/138 of 2.wav...\n",
            "\tTranscript:\t\t با مشکلاتی نظیر ازدواج و مسکن حل می شود\n",
            "\u001b[93m\tInterval Search: Matching Text: و مشکلاتی نظیر ازدواج و مسکن حل می‌شود \u001b[0m\n",
            "\u001b[93m\tInterval Search: CER: 0.05263157894736842, End Pointer: 54\u001b[0m\n",
            "\tWriting audio chunk 135...\n",
            "\tWriting text chunk 135...\n",
            "Matching chunk 136/138 of 2.wav...\n",
            "\tTranscript:\t\t و آن‌ها از فرصت برابر با سایریم برخوردار خواهند شد\n",
            "\u001b[92m\tInterval Search: Matching Text:و آنها از فرصت برابر با سایرین برخوردار خواهند شد. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.04081632653061224, End Pointer: 66\u001b[0m\n",
            "\tWriting audio chunk 136...\n",
            "\tWriting text chunk 136...\n",
            "Matching chunk 137/138 of 2.wav...\n",
            "\tTranscript:\t\t ودر پایان از شرکت‌ها و ارگان‌هایی که ن‌هایت همکاری را با ما دارند\n",
            "\u001b[92m\tInterval Search: Matching Text:در پایان از شرکت‌ها و ارگان‌هایی که نهایت همکاری را با ما دارند، \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.031746031746031744, End Pointer: 80\u001b[0m\n",
            "\tWriting audio chunk 137...\n",
            "\tWriting text chunk 137...\n",
            "Matching chunk 138/138 of 2.wav...\n",
            "\tTranscript:\t\t به ویژه سامسونگ تشکر و قدردانی میکنم\n",
            "\u001b[92m\tInterval Search: Matching Text:به ویژه سامسونگ، تشکر و قدردانی می‌کنم. \u001b[0m\n",
            "\u001b[92m\tInterval Search: CER: 0.02702702702702703, End Pointer: 55\u001b[0m\n",
            "\tWriting audio chunk 138...\n",
            "\tWriting text chunk 138...\n"
          ]
        }
      ]
    }
  ]
}