split_large_txt_file_into_smaller.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36bd579d-499e-4ac5-b730-fb963e196548",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Path to the large text file\n",
    "large_file_path = '/path/to/large/text/file.txt'\n",
    "\n",
    "# Directory to store the smaller page files\n",
    "output_directory = '/path/to/output/folder'\n",
    "\n",
    "# Create the output directory if it doesn't exist\n",
    "os.makedirs(output_directory, exist_ok=True)\n",
    "\n",
    "# Dictionary to store text content for each page\n",
    "page_content = {}\n",
    "\n",
    "# Read the large text file\n",
    "with open(large_file_path, 'r') as file:\n",
    "    for line in file:\n",
    "        line = line.strip()\n",
    "        if line:\n",
    "            # Extract volume, page number, and text content\n",
    "            parts = line.split(' ')\n",
    "            \n",
    "            # Check if line has the expected number of elements\n",
    "            if len(parts) >= 2:\n",
    "                volume = parts[0]\n",
    "                try:\n",
    "                    page = parts[1].split(':')[0]\n",
    "                except IndexError:\n",
    "                    print(f\"Skipping line: {line}\")\n",
    "                    continue\n",
    "\n",
    "                content = ' '.join(parts[2:])\n",
    "                \n",
    "                # Append content to the respective page\n",
    "                page_key = f'{volume}_{page}'\n",
    "                if page_key in page_content:\n",
    "                    page_content[page_key].append(content)\n",
    "                else:\n",
    "                    page_content[page_key] = [content]\n",
    "            else:\n",
    "                # Handle lines that don't match the expected format\n",
    "                print(f\"Skipping line: {line}\")\n",
    "\n",
    "# Write the content of each page to a separate file\n",
    "for page_key, content in page_content.items():\n",
    "    try:\n",
    "        volume, page = page_key.split('_')\n",
    "    except ValueError:\n",
    "        print(f\"Skipping invalid page key: {page_key}\")\n",
    "        continue\n",
    "    \n",
    "    output_filename = f'{volume}_{page}.txt'\n",
    "    output_file_path = os.path.join(output_directory, output_filename)\n",
    "    \n",
    "    with open(output_file_path, 'w') as file:\n",
    "        for line in content:\n",
    "            line = line.lstrip()  # Remove leading whitespace from each line\n",
    "            if line or not previous_line_empty:  # Write the line if it's not empty or if the previous line was empty\n",
    "                file.write(line + '\\n')\n",
    "            \n",
    "            previous_line_empty = not line  # Update previous_line_empty flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b7d799f-5465-4941-ba86-2226952caeca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}