Skip to content

Commit

Permalink
initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebellm
ebellm committed Oct 9, 2024
1 parent bb2057b commit 56472c6
Showing 1 changed file with 150 additions and 58 deletions.
208 changes: 150 additions & 58 deletions notebooks/AP_Data_Quality_Report.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"source": [
"# Parameters. Set defaults here.\n",
"\n",
"date = \"2024-03-29\"\n",
"date = \"2024-09-05\"\n",
"instrument = \"LATISS\""
]
},
Expand All @@ -28,24 +28,14 @@
"metadata": {},
"outputs": [],
"source": [
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3",
"metadata": {},
"outputs": [],
"source": [
"day_obs = int(date.replace(\"-\", \"\"))\n",
"day_obs = date.replace(\"-\", \"\") # day_obs is a string in the EFD\n",
"day_obs"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4",
"id": "3",
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -63,7 +53,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "5",
"id": "4",
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -80,7 +70,7 @@
},
{
"cell_type": "markdown",
"id": "6",
"id": "5",
"metadata": {},
"source": [
"1.3.14 Level 1 Data Quality Report Definition\n",
Expand All @@ -101,20 +91,31 @@
{
"cell_type": "code",
"execution_count": null,
"id": "7",
"id": "6",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from lsst_efd_client import EfdClient"
]
},
{
"cell_type": "markdown",
"id": "7",
"metadata": {},
"source": [
"As of 9 Oct 24, `lsst.prompt` is only available in usdfdev while we await authentication. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8",
"metadata": {},
"outputs": [],
"source": [
"As of 8 Feb 24, `lsst.prompt` is only available in usdfdev while we await authentication. "
"client = EfdClient(\"usdfdev_efd\", db_name=\"lsst.prompt\")"
]
},
{
Expand All @@ -124,7 +125,8 @@
"metadata": {},
"outputs": [],
"source": [
"client = EfdClient(\"usdfdev_efd\", db_name=\"lsst.prompt\")"
"# waiting on input in #sasquatch_support\n",
"# client = EfdClient(\"usdf_efd\", db_name=\"lsst.prompt\")"
]
},
{
Expand All @@ -144,12 +146,7 @@
"metadata": {},
"outputs": [],
"source": [
"query = (\n",
" f'''SELECT * FROM \"lsst.prompt.numDiaSourcesAll\"''' # where day_obs = {day_obs} '''\n",
")\n",
"print(query)\n",
"df = await client.influx_client.query(query)\n",
"df"
"await client.get_fields(\"lsst.prompt.prod.initialPviSummaryMetrics\")"
]
},
{
Expand All @@ -159,7 +156,11 @@
"metadata": {},
"outputs": [],
"source": [
"help(client.get_timeseries)"
"# Testing to find good dates\n",
"# query = f\"\"\"SELECT * FROM \"lsst.prompt.prod.initialPviSummaryMetrics\" WHERE \"day_obs\" = '{day_obs}' AND \"instrument\" = '{instrument}' \"\"\"\n",
"# print(query)\n",
"# initialPviSummaryMetrics = await client.influx_client.query(query)\n",
"# initialPviSummaryMetrics"
]
},
{
Expand All @@ -169,7 +170,10 @@
"metadata": {},
"outputs": [],
"source": [
"np.sum(df.instrument == instrument)"
"query = f\"\"\"SELECT * FROM \"lsst.prompt.prod.initialPviSummaryMetrics\" WHERE \"instrument\" = '{instrument}' \"\"\"\n",
"print(query)\n",
"initialPviSummaryMetrics = await client.influx_client.query(query)\n",
"initialPviSummaryMetrics"
]
},
{
Expand All @@ -179,10 +183,7 @@
"metadata": {},
"outputs": [],
"source": [
"query = f\"\"\"SELECT * FROM \"lsst.prompt.numDiaSourcesAll\" where instrument = {instrument}\"\"\" # where day_obs = {day_obs} '''\n",
"print(query)\n",
"df = await client.influx_client.query(query)\n",
"df"
"initialPviSummaryMetrics.columns"
]
},
{
Expand All @@ -192,7 +193,23 @@
"metadata": {},
"outputs": [],
"source": [
"df.dtypes"
"def plot_metric_vs_time_by_filter(df, column):\n",
" plt.figure()\n",
" for filt in plot_filter_colors.keys():\n",
" wf = df[\"band\"] == filt\n",
"\n",
" if np.sum(wf):\n",
" plt.plot(\n",
" df.loc[wf].index,\n",
" df.loc[wf, column],\n",
" \".\",\n",
" color=plot_filter_colors[filt],\n",
" label=filt,\n",
" )\n",
"\n",
" plt.legend()\n",
" plt.xlabel(\"Time\")\n",
" plt.ylabel(column)"
]
},
{
Expand All @@ -202,21 +219,16 @@
"metadata": {},
"outputs": [],
"source": [
"for filt in plot_filter_colors.keys():\n",
" wf = df[\"band\"] == filt\n",
"\n",
" if np.sum(wf):\n",
" plt.plot(\n",
" df.loc[wf].index,\n",
" df.loc[wf, \"numDiaSourcesAll\"],\n",
" \".\",\n",
" color=plot_filter_colors[filt],\n",
" label=filt,\n",
" )\n",
"\n",
"plt.legend()\n",
"plt.xlabel(\"Time\")\n",
"plt.ylabel(\"Number of DiaSources per detector\")"
"cols_to_plot = [\n",
" \"zeroPoint\",\n",
" \"skyBg\",\n",
" \"skyNoise\",\n",
" \"psfSigma\",\n",
" \"psfIxx\",\n",
" \"psfIxy\",\n",
" \"psfIyy\",\n",
" \"psfArea\",\n",
"]"
]
},
{
Expand All @@ -225,14 +237,19 @@
"id": "17",
"metadata": {},
"outputs": [],
"source": []
"source": [
"for col in cols_to_plot:\n",
" plot_metric_vs_time_by_filter(initialPviSummaryMetrics, col)"
]
},
{
"cell_type": "markdown",
"cell_type": "code",
"execution_count": null,
"id": "18",
"metadata": {},
"outputs": [],
"source": [
"since prompt doesn't have useful data at the moment, query the other metrics to learn how"
"plot_metric_vs_time_by_filter(initialPviSummaryMetrics, \"zeroPoint\")"
]
},
{
Expand All @@ -242,7 +259,7 @@
"metadata": {},
"outputs": [],
"source": [
"client = EfdClient(\"usdfdev_efd\", db_name=\"lsst.verify\")"
"plot_metric_vs_time_by_filter(initialPviSummaryMetrics, \"skyBg\")"
]
},
{
Expand All @@ -252,7 +269,7 @@
"metadata": {},
"outputs": [],
"source": [
"await client.get_topics()"
"plot_metric_vs_time_by_filter(initialPviSummaryMetrics, \"skyNoise\")"
]
},
{
Expand All @@ -262,9 +279,7 @@
"metadata": {},
"outputs": [],
"source": [
"query = \"\"\"SELECT * FROM \"lsst.verify.ap.totalUnassociatedDiaObjects\" \"\"\"\n",
"df = await client.influx_client.query(query)\n",
"df"
"plot_metric_vs_time_by_filter(initialPviSummaryMetrics, \"psfSigma\")"
]
},
{
Expand All @@ -273,17 +288,94 @@
"id": "22",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"metadata": {},
"outputs": [],
"source": [
"plt.plot(df.index, df.totalUnassociatedDiaObjects, \".\")"
"await client.get_fields(\"lsst.prompt.numDiaSourcesAll\")"
]
},
{
"cell_type": "markdown",
"id": "24",
"metadata": {},
"source": [
"https://docs.influxdata.com/influxdb/v1/query_language/explore-data/#the-where-clause\n",
"\n",
"> Single quote string field values in the WHERE clause. Queries with unquoted string field values or double quoted string field values will not return any data and, in most cases, will not return an error."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"id": "25",
"metadata": {},
"outputs": [],
"source": []
"source": [
"query = f\"\"\"SELECT * FROM \"lsst.prompt.numDiaSourcesAll\" WHERE \"day_obs\" = '{day_obs}' AND \"instrument\" = '{instrument}' \"\"\"\n",
"print(query)\n",
"numDiaSourcesAll = await client.influx_client.query(query)\n",
"numDiaSourcesAll"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "26",
"metadata": {},
"outputs": [],
"source": [
"numDiaSourcesAll.columns"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "27",
"metadata": {},
"outputs": [],
"source": [
"numDiaSourcesAll.iloc[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28",
"metadata": {},
"outputs": [],
"source": [
"# help(client.select_time_series)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "29",
"metadata": {},
"outputs": [],
"source": [
"for filt in plot_filter_colors.keys():\n",
" wf = numDiaSourcesAll[\"band\"] == filt\n",
"\n",
" if np.sum(wf):\n",
" plt.plot(\n",
" numDiaSourcesAll.loc[wf].index,\n",
" numDiaSourcesAll.loc[wf, \"numDiaSourcesAll\"],\n",
" \".\",\n",
" color=plot_filter_colors[filt],\n",
" label=filt,\n",
" )\n",
"\n",
"plt.legend()\n",
"plt.xlabel(\"Time\")\n",
"plt.ylabel(\"Number of DiaSources per detector\")"
]
}
],
"metadata": {
Expand All @@ -302,7 +394,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
"version": "3.11.9"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 56472c6

Please sign in to comment.