cratedb-wtf: Add cratedb-wtf diagnostics program

.gitignore

@@ -7,3 +7,5 @@ __pycache__
 dist
 .coverage*
 coverage.xml
+/foo
+/tmp

CHANGES.md

@@ -14,6 +14,7 @@
 - MongoDB: Improve UX by using `ctk load table mongodb://...`
 - load table: Refactor to use more OO
 - Add `examples/cloud_import.py`
+- Add `cratedb-wtf` diagnostics program
 
 
 ## 2023/11/06 v0.0.2

cratedb_toolkit/cli.py

@@ -7,6 +7,7 @@
 from .io.cli import cli as io_cli
 from .job.cli import cli_list_jobs
 from .shell.cli import cli as shell_cli
+from .wtf.cli import cli as wtf_cli
 
 
 @click.group(cls=ClickAliasedGroup)  # type: ignore[arg-type]
@@ -21,4 +22,5 @@ def cli(ctx: click.Context, verbose: bool, debug: bool):
 cli.add_command(cloud_cli, name="cluster")
 cli.add_command(io_cli, name="load")
 cli.add_command(shell_cli, name="shell")
+cli.add_command(wtf_cli, name="wtf")
 cli.add_command(cli_list_jobs)

cratedb_toolkit/util/cli.py

@@ -72,13 +72,15 @@ def boot_with_dburi():
     return dburi
 
 
-def make_command(cli, name, helpfun=None, aliases=None):
+def make_command(cli, name, help=None, aliases=None):  # noqa: A002
     """
     Convenience shortcut for creating a subcommand.
     """
     kwargs = {}
-    if helpfun:
-        kwargs["help"] = docstring_format_verbatim(helpfun.__doc__)
+    if isinstance(help, str):
+        kwargs["help"] = help
+    elif callable(help):
+        kwargs["help"] = docstring_format_verbatim(help.__doc__)
     return cli.command(
         name,
         context_settings={"max_content_width": 120},

cratedb_toolkit/util/data.py

@@ -1,3 +1,4 @@
+import datetime as dt
 import json
 import sys
 import typing as t
@@ -7,7 +8,7 @@ def jd(data: t.Any):
     """
     Pretty-print JSON with indentation.
     """
-    print(json.dumps(data, indent=2), file=sys.stdout)  # noqa: T201
+    print(json.dumps(data, indent=2, cls=JSONEncoderPlus), file=sys.stdout)  # noqa: T201
 
 
 def str_contains(haystack, *needles):
@@ -16,3 +17,15 @@ def str_contains(haystack, *needles):
     """
     haystack = str(haystack)
     return any(needle in haystack for needle in needles)
+
+
+class JSONEncoderPlus(json.JSONEncoder):
+    """
+    https://stackoverflow.com/a/27058505
+    """
+
+    def default(self, o):
+        if isinstance(o, dt.datetime):
+            return o.isoformat()
+
+        return json.JSONEncoder.default(self, o)

cratedb_toolkit/util/platform.py

@@ -0,0 +1,56 @@
+import io
+import json
+from contextlib import redirect_stdout
+
+
+class PlatformInfo:
+    @staticmethod
+    def application():
+        import platform
+
+        from cratedb_toolkit import __appname__, __version__
+
+        data = {}
+
+        data["platform"] = platform.platform()
+        data["version"] = __version__
+        data["name"] = __appname__
+        return data
+
+    @staticmethod
+    def libraries():
+        data = {}
+
+        # SQLAlchemy
+        from importlib.metadata import entry_points
+
+        try:
+            import sqlalchemy.dialects.plugins
+            import sqlalchemy.dialects.registry
+
+            data["sqlalchemy"] = {
+                "dialects_builtin": list(sqlalchemy.dialects.registry.impls.keys()),
+                "dialects_3rdparty": [dialect.name for dialect in entry_points(group="sqlalchemy.dialects")],  # type: ignore[attr-defined,call-arg]
+                "plugins": list(sqlalchemy.dialects.plugins.impls.keys()),
+            }
+        except Exception:  # noqa: S110
+            pass
+
+        # pandas
+        try:
+            import pandas
+
+            buffer = io.StringIO()
+            with redirect_stdout(buffer):
+                pandas.show_versions(as_json=True)
+            buffer.seek(0)
+            data["pandas"] = json.load(buffer)
+        except Exception:  # noqa: S110
+            pass
+
+        # fsspec
+        import fsspec
+
+        data["fsspec"] = {"protocols": fsspec.available_protocols(), "compressions": fsspec.available_compressions()}
+
+        return data

cratedb_toolkit/util/service.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2021-2023, Crate.io Inc.
+# Distributed under the terms of the AGPLv3 license, see LICENSE.
+import logging
+import typing as t
+
+from cratedb_toolkit.util.common import setup_logging
+
+logger = logging.getLogger(__name__)
+
+
+def start_service(app: str, listen_address: t.Union[str, None] = None, reload: bool = False):  # pragma: no cover
+    setup_logging()
+    from uvicorn import run
+
+    if listen_address is None:
+        listen_address = "127.0.0.1:4242"
+
+    host, port = listen_address.split(":")
+    port_int = int(port)
+
+    logger.info(f"Starting HTTP web service on http://{listen_address}")
+
+    run(app=app, host=host, port=port_int, reload=reload)

cratedb_toolkit/wtf/README.md

@@ -0,0 +1,51 @@
+# cratedb-wtf
+
+A diagnostics utility in the spirit of [git-wtf], [grafana-wtf], and [pip.wtf].
+It is still a work-in-progress, but it is usable already.
+
+
+## Synopsis
+
+Define CrateDB database cluster address.
+```shell
+export CRATEDB_SQLALCHEMY_URL=crate://localhost/
+```
+
+Display system and database cluster information.
+```shell
+cratedb-wtf info
+```
+
+Display database cluster job information.
+```shell
+cratedb-wtf job-info
+```
+
+Display database cluster log messages.
+```shell
+cratedb-wtf logs
+```
+
+Statistics.
+```shell
+cratedb-wtf job-statistics collect
+cratedb-wtf job-statistics view
+```
+
+
+## HTTP API
+
+Expose collected status information. 
+```shell
+cratedb-wtf --debug serve --reload
+```
+Consume collected status information via HTTP.
+```shell
+http http://127.0.0.1:4242/info/all
+```
+
+
+
+[git-wtf]: http://thrawn01.org/posts/2014/03/03/git-wtf/ 
+[grafana-wtf]: https://github.com/panodata/grafana-wtf
+[pip.wtf]: https://github.com/sabslikesobs/pip.wtf

cratedb_toolkit/wtf/backlog.md

@@ -0,0 +1,34 @@
+# cratedb-wtf backlog
+
+## Iteration +1
+- Expose collected data via Glances-like UI
+- Experimental UI using Grafana Scenes
+
+## Iteration +2
+- Make `cratedb-wtf logs` also optionally consider `sys.` tables. 
+- cratedb-wtf explore table|shard|partition|node
+- High-level analysis, evaluating a set of threshold rules 
+- Network diagnostics?
+
+## Iteration +3
+- Make it work with CrateDB Cloud.
+  ```
+  ctk cluster info
+  ctk cluster health
+  ctk cluster logs --slow-queries
+  ```
+
+## Done
+- Make it work
+- Proper marshalling of timestamp values (ISO 8601)
+- Expose collected data via HTTP API
+  ```
+  cratedb-wtf serve
+  ```
+- Provide `scrub` option also via HTTP
+- Complete collected queries and code snippets
+- Harvest queries from Admin UI, crash, crate-admin-alt
+- Harvest queries from experts
+  - https://tools.cr8.net/grafana/d/RkpNJx84z/cratedb-jobs-log?orgId=1&refresh=5m&var-datasource=crate-production
+  - https://tools.cr8.net/grafana/d/RkpNJx84z/cratedb-jobs-log?orgId=1&refresh=5m&var-datasource=crate-production&viewPanel=44
+- Add `description` and `unit` fields to each `InfoElement` definition