typing, refs, update head ts

docs/7.references/4.models.md

@@ -399,48 +399,6 @@ description: "Models reference"
 
 ## Substrate
 
-<dl class="py class">
-
-## dipdup.models.substrate.SubstrateEvent
-
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.substrate.</span></span><span class="sig-name descname"><span class="pre">SubstrateEvent</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">data</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#dipdupmodelssubstratesubstrateeventdata" title="dipdup.models.substrate.SubstrateEventData" target="_self"><span class="pre">dipdup.models.substrate.SubstrateEventData</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">runtime</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dipdup.runtimes.SubstrateRuntime</span></span></em><span class="sig-paren">)</span></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>data</strong> (<a class="reference internal" href="#dipdupmodelssubstratesubstrateeventdata" title="dipdup.models.substrate.SubstrateEventData" target="_self"><em>SubstrateEventData</em></a>)</p></li>
-<li><p><strong>runtime</strong> (<em>SubstrateRuntime</em>)</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="py class">
-
-## dipdup.models.substrate.SubstrateEventData
-
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.substrate.</span></span><span class="sig-name descname"><span class="pre">SubstrateEventData</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">index</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">extrinsicIndex</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">callAddress</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">args</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">list</span><span class="p"><span class="pre">[</span></span><span class="pre">dipdup.fields.Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">decoded_args</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">dipdup.fields.Any</span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">header</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#dipdupmodelssubstrateblockheader" title="dipdup.models.substrate.BlockHeader" target="_self"><span class="pre">dipdup.models.substrate.BlockHeader</span></a></span></em><span class="sig-paren">)</span></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd" style="color: var(--txt-primary);">Parameters<span class="colon">:</span></dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>name</strong> (<em>str</em>)</p></li>
-<li><p><strong>index</strong> (<em>int</em>)</p></li>
-<li><p><strong>extrinsicIndex</strong> (<em>int</em>)</p></li>
-<li><p><strong>callAddress</strong> (<em>list</em><em>[</em><em>str</em><em>] </em><em>| </em><em>None</em>)</p></li>
-<li><p><strong>args</strong> (<em>list</em><em>[</em><em>Any</em><em>] </em><em>| </em><em>None</em>)</p></li>
-<li><p><strong>decoded_args</strong> (<em>dict</em><em>[</em><em>str</em><em>, </em><em>Any</em><em>] </em><em>| </em><em>None</em>)</p></li>
-<li><p><strong>header</strong> (<a class="reference internal" href="#dipdupmodelssubstrateblockheader" title="dipdup.models.substrate.BlockHeader" target="_self"><em>BlockHeader</em></a>)</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="py class">
-
-## dipdup.models.substrate.BlockHeader
-
-<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">dipdup.models.substrate.</span></span><span class="sig-name descname"><span class="pre">BlockHeader</span></span></dt>
-<dd></dd></dl>
-
 </section>
 
 

docs/models.rst

@@ -55,7 +55,7 @@ Substrate
 
 .. autoclass:: dipdup.models.substrate.SubstrateEvent
 .. autoclass:: dipdup.models.substrate.SubstrateEventData
-.. autoclass:: dipdup.models.substrate.BlockHeader
+.. autoclass:: dipdup.models.substrate.SubstrateHeadBlockData
 
 -------------------------------------------------------------------------------
 Tezos

scripts/docs.py

@@ -145,7 +145,6 @@ class ReferencePage(TypedDict):
     'dipdup.models.QuerySet',
     'dipdup.models.RollbackMessage',
     'dipdup.models.substrate.HeadBlock',
-    'dipdup.models.substrate.SubstrateEventDataDict',
     'dipdup.models.substrate_node.SubstrateNodeHeadSubscription',
     'dipdup.models.subsquid.AbstractSubsquidQuery',
     'dipdup.models.subsquid.SubsquidMessageType',
@@ -517,7 +516,7 @@ def _compare(ref: str, ignore: set[str]) -> None:
                 else:
                     package_path_str = '.' + package_path.with_suffix('').as_posix().replace('/', '.')
                 # NOTE: Skip private modules and classes
-                if '._' in package_path_str or 'ABC' in match.group(2):
+                if '._' in package_path_str or 'ABC' in match.group(2) or match.group(1)[0] == '_':
                     continue
                 classes_in_package.add(f'dipdup.{ref}{package_path_str}.{match.group(1)}')
 

src/dipdup/config/__init__.py

@@ -287,7 +287,8 @@ class DatasourceConfig(ABC, NameMixin):
     """
 
     kind: str
-    url: str
+    url: Url
+    ws_url: WsUrl | None = None
     http: HttpConfig | None = None
 
 

src/dipdup/datasources/__init__.py

@@ -143,8 +143,7 @@ def _get_ws_client(self) -> WebsocketTransport:
 
         self._logger.debug('Creating Websocket client')
 
-        # FIXME: correct config class
-        url = self._config.ws_url  # type: ignore
+        url = self._config.ws_url
         if not url:
             raise FrameworkException('Spawning node datasource, but `ws_url` is not set')
         self._ws_client = WebsocketTransport(

src/dipdup/datasources/substrate_node.py

@@ -19,10 +19,10 @@
 from dipdup.datasources import JsonRpcDatasource
 from dipdup.exceptions import DatasourceError
 from dipdup.exceptions import FrameworkException
-from dipdup.models.substrate import BlockHeader
-from dipdup.models.substrate import HeadBlock
 from dipdup.models.substrate import SubstrateEventData
-from dipdup.models.substrate import SubstrateEventDataDict
+from dipdup.models.substrate import SubstrateHeadBlockData
+from dipdup.models.substrate import _BlockHeader
+from dipdup.models.substrate import _SubstrateNodeEventResponse
 from dipdup.models.substrate_node import SubstrateNodeHeadSubscription
 from dipdup.models.substrate_node import SubstrateNodeSubscription
 from dipdup.pysignalr import Message
@@ -32,14 +32,14 @@
 _logger = logging.getLogger(__name__)
 
 
-HeadCallback = Callable[['SubstrateNodeDatasource', HeadBlock], Awaitable[None]]
+HeadCallback = Callable[['SubstrateNodeDatasource', SubstrateHeadBlockData], Awaitable[None]]
 EventCallback = Callable[['SubstrateNodeDatasource', tuple[SubstrateEventData, ...]], Awaitable[None]]
 
 
 # NOTE: Renamed entity class LevelData from evm_node
 @dataclass
 class SubscriptionMessage:
-    head: HeadBlock
+    head: SubstrateHeadBlockData
     fetch_events: bool = False
 
 
@@ -168,7 +168,7 @@ async def subscribe(self) -> None:
             if isinstance(subscription, SubstrateNodeSubscription):
                 await self._subscribe(subscription)
 
-    async def emit_head(self, head: HeadBlock) -> None:
+    async def emit_head(self, head: SubstrateHeadBlockData) -> None:
         for fn in self._on_head_callbacks:
             await fn(self, head)
 
@@ -221,7 +221,7 @@ async def get_head_level(self) -> int:
     async def get_block_hash(self, height: int) -> str:
         return await self._jsonrpc_request('chain_getBlockHash', [height])  # type: ignore[no-any-return]
 
-    async def get_block_header(self, hash: str) -> BlockHeader:
+    async def get_block_header(self, hash: str) -> _BlockHeader:
         response = await self._jsonrpc_request('chain_getHeader', [hash])
         # FIXME: missing fields
         return {
@@ -246,10 +246,10 @@ async def get_metadata_header_batch(self, heights: list[int]) -> list[MetadataHe
     async def get_full_block(self, hash: str) -> dict[str, Any]:
         return await self._jsonrpc_request('chain_getBlock', [hash])  # type: ignore[no-any-return]
 
-    async def get_events(self, block_hash: str) -> tuple[SubstrateEventDataDict, ...]:
+    async def get_events(self, block_hash: str) -> tuple[_SubstrateNodeEventResponse, ...]:
         events = await self._interface.get_events(block_hash)
 
-        result: list[SubstrateEventDataDict] = []
+        result: list[_SubstrateNodeEventResponse] = []
         for raw_event in events:
             event: dict[str, Any] = raw_event.decode()
             result.append(

src/dipdup/datasources/substrate_subsquid.py

@@ -3,7 +3,7 @@
 from dipdup.config.substrate_subsquid import SubstrateSubsquidDatasourceConfig
 from dipdup.datasources._subsquid import AbstractSubsquidDatasource
 from dipdup.models._subsquid import AbstractSubsquidQuery
-from dipdup.models.substrate import SubstrateEventDataSubsquid
+from dipdup.models.substrate import _SubstrateSubsquidEventResponse
 
 Query = AbstractSubsquidQuery
 
@@ -14,7 +14,7 @@ async def iter_events(
         first_level: int,
         last_level: int,
         names: tuple[str, ...],
-    ) -> AsyncIterator[tuple[SubstrateEventDataSubsquid, ...]]:
+    ) -> AsyncIterator[tuple[_SubstrateSubsquidEventResponse, ...]]:
         current_level = first_level
 
         while current_level <= last_level:

src/dipdup/dipdup.py

@@ -74,8 +74,8 @@
 from dipdup.models.evm import EvmTransactionData
 from dipdup.models.evm_node import EvmNodeHeadData
 from dipdup.models.evm_node import EvmNodeSyncingData
-from dipdup.models.substrate import HeadBlock
 from dipdup.models.substrate import SubstrateEventData
+from dipdup.models.substrate import SubstrateHeadBlockData
 from dipdup.models.tezos import TezosBigMapData
 from dipdup.models.tezos import TezosEventData
 from dipdup.models.tezos import TezosHeadBlockData
@@ -560,10 +560,10 @@ async def _on_tzkt_events(self, datasource: TezosTzktDatasource, events: tuple[T
     async def _on_substrate_head(
         self,
         datasource: SubstrateNodeDatasource,
-        head: HeadBlock,
+        head: SubstrateHeadBlockData,
     ) -> None:
-        # TODO: any head updates here?
-        pass
+        # TODO: update Head. Does fire_and_forget work atm?
+        metrics._datasource_head_updated[datasource.name] = time.time()
 
     async def _on_substrate_events(
         self,

src/dipdup/models/substrate.py

@@ -11,7 +11,7 @@
 from dipdup.runtimes import SubstrateRuntime
 
 
-class BlockHeaderSubsquid(TypedDict):
+class _BlockHeaderExtra(TypedDict):
     number: int
     hash: str
     parentHash: str
@@ -26,22 +26,22 @@ class BlockHeaderSubsquid(TypedDict):
     validator: str
 
 
-class SubstrateEventDataSubsquid(TypedDict):
+class _SubstrateSubsquidEventResponse(TypedDict):
     name: str
     index: int
     extrinsicIndex: int
     callAddress: list[str]
     args: list[Any]
-    header: BlockHeaderSubsquid
+    header: _BlockHeaderExtra
 
 
-class BlockHeader(TypedDict):
+class _BlockHeader(TypedDict):
     hash: str
     number: int
     prev_root: str
 
 
-class SubstrateEventDataDict(TypedDict):
+class _SubstrateNodeEventResponse(TypedDict):
     name: str
     index: int
     extrinsic_index: int
@@ -58,15 +58,15 @@ class SubstrateEventData(HasLevel):
     # we receive decoded args from node datasource and encoded from subsquid datasource
     args: list[Any] | None = None
     decoded_args: dict[str, Any] | None = None
-    header: BlockHeader
-    header_extra: BlockHeaderSubsquid | None
+    header: _BlockHeader
+    header_extra: _BlockHeaderExtra | None
 
     @property
     def level(self) -> int:  # type: ignore[override]
         return self.header['number']
 
     @classmethod
-    def from_node(cls, event_dict: SubstrateEventDataDict, header: BlockHeader) -> Self:
+    def from_node(cls, event_dict: _SubstrateNodeEventResponse, header: _BlockHeader) -> Self:
         return cls(
             **event_dict,
             call_address=None,
@@ -76,7 +76,7 @@ def from_node(cls, event_dict: SubstrateEventDataDict, header: BlockHeader) -> S
         )
 
     @classmethod
-    def from_subsquid(cls, event_dict: SubstrateEventDataSubsquid) -> Self:
+    def from_subsquid(cls, event_dict: _SubstrateSubsquidEventResponse) -> Self:
         return cls(
             name=event_dict['name'],
             index=event_dict['index'],
@@ -93,7 +93,7 @@ def from_subsquid(cls, event_dict: SubstrateEventDataSubsquid) -> Self:
         )
 
 
-class HeadBlock(TypedDict):
+class SubstrateHeadBlockData(TypedDict):
     parentHash: str
     number: str
     stateRoot: str
@@ -109,13 +109,14 @@ class SubstrateEvent(Generic[PayloadT]):
     data: SubstrateEventData
     runtime: SubstrateRuntime
 
-    # TODO: could be used in other models with typed payload
+    # TODO: Use lazy decoding in other models with typed payload
     @cached_property
     def payload(self) -> PayloadT:
+        # NOTE: from node datasource
         if self.data.decoded_args is not None:
             return cast(PayloadT, self.data.decoded_args)
 
-        # NOTE: both from subsquid
+        # NOTE: from subsquid datasource
         assert self.data.args is not None and self.data.header_extra is not None
         return cast(
             PayloadT,