Utilities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8427cd9",
   "metadata": {},
   "source": [
    "# Utilities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7dc1ded",
   "metadata": {},
   "source": [
    "Set dataset path and scale applied to spines during visualization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96caf759",
   "metadata": {},
   "outputs": [],
   "source": [
    "from notebook_widgets import SpineMeshDataset\n",
    "\n",
    "# set dataset path and scale applied to spines during visualization\n",
    "dataset_path = \"example_dendrite\"\n",
    "scale = (1, 1, 1)\n",
    "\n",
    "# load and scale spine mesh dataset\n",
    "spine_dataset = SpineMeshDataset().load(dataset_path)\n",
    "scaled_spine_dataset = SpineMeshDataset().load(dataset_path)\n",
    "scaled_spine_dataset.apply_scale(scale)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e811373b",
   "metadata": {},
   "source": [
    "### View Dendrite Skeleton"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2cdd285",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from notebook_widgets import view_skeleton_widget\n",
    "\n",
    "\n",
    "display(view_skeleton_widget(scaled_spine_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4928979",
   "metadata": {},
   "source": [
    "### View Dendrite segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abb73afc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from notebook_widgets import dendrite_segmentation_view_widget\n",
    "\n",
    "display(dendrite_segmentation_view_widget(scaled_spine_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cb42ad8",
   "metadata": {},
   "source": [
    "### View Chords\n",
    "Set `num_of_chords` and `num_of_bins`. Histograms can be exported to `dataset_path/chords_%num_of_chords%_chords_%num_of_bins%_bins.csv` file. Histograms are only exported for meshes that were viewed! Otherwise it would take too much time to calculate. Use `Calculate Metrics` cell to calculate metrics for the entire dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d04d99d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from notebook_widgets import spine_chords_widget\n",
    "\n",
    "\n",
    "num_of_chords = 30000\n",
    "num_of_bins = 100\n",
    "\n",
    "display(spine_chords_widget(spine_dataset, scaled_spine_dataset, dataset_path, num_of_chords, num_of_bins))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39e23614",
   "metadata": {},
   "source": [
    "### Calculate Metrics\n",
    "Metrics for the dataset will be saved to `dataset_path/metrics.csv`. Chords histograms will be saved separately to `dataset_path/chords.csv`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37a6cc2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spine_metrics import SpineMetricDataset\n",
    "\n",
    "\n",
    "# chord method parameters\n",
    "num_of_chords = 30000\n",
    "num_of_bins = 100\n",
    "\n",
    "# calculate metrics\n",
    "metric_names = [\"OldChordDistribution\", \"OpenAngle\", \"CVD\", \"AverageDistance\",\n",
    "                \"LengthVolumeRatio\", \"LengthAreaRatio\", \"JunctionArea\",\n",
    "                \"Length\", \"Area\", \"Volume\", \"ConvexHullVolume\", \"ConvexHullRatio\"]\n",
    "metric_params = [{\"num_of_chords\": num_of_chords, \"num_of_bins\": num_of_bins},\n",
    "                 {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}]\n",
    "spine_metrics = SpineMetricDataset()\n",
    "spine_metrics.calculate_metrics(spine_dataset.spine_meshes, metric_names, metric_params)\n",
    "spine_metrics.save(f\"{dataset_path}/metrics.csv\")    \n",
    "\n",
    "spine_metrics.get_metrics_subset([\"OldChordDistribution\"]).save_as_array(f\"{dataset_path}/chords.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da66dc0b",
   "metadata": {},
   "source": [
    "### View Spines in Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be4e093a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from spine_metrics import SpineMetricDataset\n",
    "from notebook_widgets import spine_dataset_view_widget\n",
    "\n",
    "\n",
    "metrics = SpineMetricDataset().load(f\"{dataset_path}/metrics.csv\")\n",
    "display(spine_dataset_view_widget(scaled_spine_dataset, metrics))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f04146f7",
   "metadata": {},
   "source": [
    "### Merge Manual Classifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33ed3fc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spine_metrics import SpineMetricDataset\n",
    "from notebook_widgets import create_dir, remove_file\n",
    "from spine_fitter import SpineGrouping\n",
    "import pathlib\n",
    "\n",
    "dataset_path = \"0.025 0.025 0.1 dataset\"\n",
    "merged_path = f\"{dataset_path}/manual_classification/manual_classification_merged.json\"\n",
    "merged_reduced_path = f\"{dataset_path}/manual_classification/manual_classification_merged_reduced.json\"\n",
    "\n",
    "# remove old merged classifications\n",
    "remove_file(merged_path)\n",
    "remove_file(merged_reduced_path)\n",
    "\n",
    "# load manual classifications\n",
    "path = pathlib.Path(f\"{dataset_path}/manual_classification\")\n",
    "classification_paths = [str(classification_path) for classification_path in path.glob(\"*.json\")]\n",
    "groupings = [SpineGrouping().load(path) for path in classification_paths]\n",
    "print(f\"Merging manual classifications: {classification_paths}\\n\")\n",
    "\n",
    "# merge classifications\n",
    "merged_grouping = SpineGrouping.merge(groupings, outliers_label=\"Unclassified\")\n",
    "merged_grouping.save(merged_path)\n",
    "print(f\"Saved to \\\"{merged_path}\\\".\")\n",
    "\n",
    "# remove spines with no consensus, filopodias and labeled-as-outliers\n",
    "contested = SpineGrouping.get_contested_samples(groupings)\n",
    "merged_grouping.remove_samples(contested)\n",
    "merged_grouping.remove_samples(merged_grouping.groups[\"Filopodia\"].copy())\n",
    "merged_grouping.remove_samples(merged_grouping.groups[\"Outlier\"].copy())\n",
    "del merged_grouping.groups[\"Filopodia\"]\n",
    "del merged_grouping.groups[\"Outlier\"]\n",
    "\n",
    "merged_grouping.save(merged_reduced_path)\n",
    "print(f\"Saved to \\\"{merged_reduced_path}\\\".\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70096a74",
   "metadata": {},
   "source": [
    "### Consensus Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c0654ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "from notebook_widgets import consensus_widget\n",
    "from spine_fitter import SpineGrouping\n",
    "import pathlib\n",
    "\n",
    "# load manual classifications\n",
    "path = pathlib.Path(f\"{dataset_path}/manual_classification\")\n",
    "classification_paths = path.glob(\"manual_classification_?.json\")\n",
    "groupings = [SpineGrouping().load(str(path)) for path in classification_paths]\n",
    "\n",
    "# show consensus table\n",
    "display(consensus_widget(groupings))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73b32535",
   "metadata": {},
   "source": [
    "### Merge Train & Test Manual Classifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58390623",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spine_fitter import SpineGrouping\n",
    "import pathlib\n",
    "\n",
    "\n",
    "def remove_prefix(grouping: SpineGrouping, prefix: str) -> None:\n",
    "    for label, group in grouping.groups.items():\n",
    "        new_group = set()\n",
    "        for spine_name in group:\n",
    "            new_group.add(spine_name.replace(prefix, \"\"))\n",
    "        grouping.groups[label] = new_group\n",
    "\n",
    "    new_samples = set()    \n",
    "    for spine_name in grouping.samples:\n",
    "        new_samples.add(spine_name.replace(prefix, \"\"))\n",
    "    grouping.samples = new_samples\n",
    "\n",
    "\n",
    "path = pathlib.Path(f\"test 0.025 0.025 0.1 dataset/manual_classification\")\n",
    "classification_paths = path.glob(\"manual_classification_?.json\")\n",
    "\n",
    "for classification_path in classification_paths:\n",
    "    grouping_test = SpineGrouping().load(str(classification_path))\n",
    "    grouping_train = SpineGrouping().load(f\"train 0.025 0.025 0.1 dataset/manual_classification/{classification_path.name}\")\n",
    "\n",
    "    remove_prefix(grouping_train, \"train \")\n",
    "    remove_prefix(grouping_test, \"test \")\n",
    "\n",
    "    merged = SpineGrouping.merge([grouping_train, grouping_test])\n",
    "    merged.save(f\"0.025 0.025 0.1 dataset/manual_classification/{classification_path.name}\")\n",
    "    \n",
    "    print(f'Saved classification to \"0.025 0.025 0.1 dataset/manual_classification/{classification_path.name}\"')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}