save scratch work

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.6.1
+    rev: 0.7.1
     hooks:
       - id: nbstripout
   - repo: https://github.com/psf/black
-    rev: "22.1.0"
+    rev: "24.4.0"
     hooks:
       - id: black-jupyter

notebooks/AP_Data_Quality_Report.ipynb

@@ -0,0 +1,310 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0",
+   "metadata": {},
+   "source": [
+    "# Alert Production Data Quality Report for {{params.instrument}} on {{ params.date }}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Parameters.  Set defaults here.\n",
+    "\n",
+    "date = \"2024-03-29\"\n",
+    "instrument = \"LATISS\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "day_obs = int(date.replace(\"-\", \"\"))\n",
+    "day_obs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "match instrument:\n",
+    "    case \"LATISS\":\n",
+    "        sal_index = 2\n",
+    "        n_detector = 1\n",
+    "    case \"LSSTComCamSim\":\n",
+    "        sal_index = 3\n",
+    "        n_detector = 9\n",
+    "    case _:\n",
+    "        logger.error(f\"Unknown instrument {instrument}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# https://rtn-045.lsst.io/#colorblind-friendly-plots\n",
+    "plot_filter_colors = {\n",
+    "    \"u\": \"#56b4e9\",\n",
+    "    \"g\": \"#008060\",\n",
+    "    \"r\": \"#ff4000\",\n",
+    "    \"i\": \"#850000\",\n",
+    "    \"z\": \"#6600cc\",\n",
+    "    \"y\": \"#000000\",\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "1.3.14 Level 1 Data Quality Report Definition\n",
+    "\n",
+    "ID: DMS-REQ-0097 (Priority: 1a)\n",
+    "\n",
+    "Specification: The DMS shall produce a Level 1 Data Quality Report that contains indicators\n",
+    "of data quality that result from running the DMS pipelines, including at least: Photometric\n",
+    "zero point vs. time for each utilized filter; Sky brightness vs. time for each utilized filter; seeing\n",
+    "vs. time for each utilized filter; PSF parameters vs. time for each utilized filter; detection\n",
+    "efficiency for point sources vs. mag for each utilized filter.\n",
+    "\n",
+    "Discussion: The seeing report is intended as a broad-brush measure of image quality. The\n",
+    "PSF parameters provide more detail, as they include asymmetries and field location\n",
+    "dependence"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "from lsst_efd_client import EfdClient"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
+   "source": [
+    "As of 8 Feb 24, `lsst.prompt` is only available in usdfdev while we await authentication. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = EfdClient(\"usdfdev_efd\", db_name=\"lsst.prompt\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "await client.get_topics()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = (\n",
+    "    f'''SELECT * FROM \"lsst.prompt.numDiaSourcesAll\"'''  # where day_obs = {day_obs} '''\n",
+    ")\n",
+    "print(query)\n",
+    "df = await client.influx_client.query(query)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "help(client.get_timeseries)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "np.sum(df.instrument == instrument)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = f\"\"\"SELECT * FROM \"lsst.prompt.numDiaSourcesAll\" where instrument = {instrument}\"\"\"  # where day_obs = {day_obs} '''\n",
+    "print(query)\n",
+    "df = await client.influx_client.query(query)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.dtypes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for filt in plot_filter_colors.keys():\n",
+    "    wf = df[\"band\"] == filt\n",
+    "\n",
+    "    if np.sum(wf):\n",
+    "        plt.plot(\n",
+    "            df.loc[wf].index,\n",
+    "            df.loc[wf, \"numDiaSourcesAll\"],\n",
+    "            \".\",\n",
+    "            color=plot_filter_colors[filt],\n",
+    "            label=filt,\n",
+    "        )\n",
+    "\n",
+    "plt.legend()\n",
+    "plt.xlabel(\"Time\")\n",
+    "plt.ylabel(\"Number of DiaSources per detector\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "18",
+   "metadata": {},
+   "source": [
+    "since prompt doesn't have useful data at the moment, query the other metrics to learn how"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "client = EfdClient(\"usdfdev_efd\", db_name=\"lsst.verify\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "await client.get_topics()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"\"\"SELECT * FROM \"lsst.verify.ap.totalUnassociatedDiaObjects\" \"\"\"\n",
+    "df = await client.influx_client.query(query)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "plt.plot(df.index, df.totalUnassociatedDiaObjects, \".\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "LSST",
+   "language": "python",
+   "name": "lsst"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/AP_Data_Quality_Report.yaml

@@ -0,0 +1,10 @@
+title: AP Data Quality Report
+description: Data Quality Metrics Computed by AP
+authors:
+  - name: Eric Bellm
+    slack: ebellm
+parameters:
+  date:
+    type: string
+    description: Night (YYYY-MM-DD)
+    default: "2024-02-01"