From 513c430243947051075005b01699b423ba2deff3 Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Wed, 24 Jan 2024 18:11:31 +0100 Subject: [PATCH 1/8] wl --- mpyc-web-py/benches/timeouts.py | 6 ++--- mpyc-web-py/lib/rstats/rstats/stats.py | 3 +++ mpyc-web-py/lib/stats.py | 15 +++++++----- mpyc-web-py/lib/webloop_pyodide_old.py | 12 +++++----- mpyc-web-py/lib/weblooper.py | 2 ++ mpyc-web-py/lib/weblooperV10.py | 33 +++++++++++++++++++++++--- 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/mpyc-web-py/benches/timeouts.py b/mpyc-web-py/benches/timeouts.py index 876505f5..9827c205 100644 --- a/mpyc-web-py/benches/timeouts.py +++ b/mpyc-web-py/benches/timeouts.py @@ -93,10 +93,10 @@ def loopz(a: Any, b: Any, c: Any = 4): async def main(): await async_sleep_0() - sync_sleep_0() + # sync_sleep_0() - await async_nothing() - sync_nothing() + # await async_nothing() + # sync_nothing() logging.info("---done---") diff --git a/mpyc-web-py/lib/rstats/rstats/stats.py b/mpyc-web-py/lib/rstats/rstats/stats.py index cef6d148..9fc3bd1e 100644 --- a/mpyc-web-py/lib/rstats/rstats/stats.py +++ b/mpyc-web-py/lib/rstats/rstats/stats.py @@ -73,6 +73,9 @@ def __len__(self): def __float__(self): return self._total / len(self._ring) + def __str__(self): + return str(float(self)) + class DeepCounter(NestedDict[K, Numeric | Any]): """A nested dictionary that stores numeric values and supports recursive updates. diff --git a/mpyc-web-py/lib/stats.py b/mpyc-web-py/lib/stats.py index e63f2a15..0140db14 100644 --- a/mpyc-web-py/lib/stats.py +++ b/mpyc-web-py/lib/stats.py @@ -153,6 +153,9 @@ def to_tree(self): return super().to_tree() def asyncio_stats(self): + if "asyncio" not in self.stats: + return + tasks = len(asyncio.tasks._all_tasks) if tasks > self.stats["asyncio"]["max_tasks"]: self.stats["asyncio"]["max_tasks"] = tasks @@ -202,21 +205,21 @@ def stat(self, s: NestedDict[str, float]) -> NestedDict[str, float]: """ return s - def latency(self, ts: int) -> NestedDict[str, float]: # pyright: ignore + def latency(self, ts: int, key="latency") -> NestedDict[str, float]: # pyright: ignore l: int = time.time_ns() // 1000 - ts if l <= 0: return {} - if "latency" not in self.stats: - self.stats["latency"] = { # pyright: ignore + if key not in self.stats: + self.stats[key] = { # pyright: ignore "min": None, "max": 0, "avg": MovingAverage(maxlen=200), } - self.stats["latency"]["avg"].append(l) - self.stats["latency"]["min"] = min(l, self.stats["latency"]["min"]) if self.stats["latency"]["min"] else l # pyright: ignore - self.stats["latency"]["max"] = max(l, self.stats["latency"]["max"]) if self.stats["latency"]["max"] else l # pyright: ignore + self.stats[key]["avg"].append(l) + self.stats[key]["min"] = min(l, self.stats[key]["min"]) if self.stats[key]["min"] else l # pyright: ignore + self.stats[key]["max"] = max(l, self.stats[key]["max"]) if self.stats[key]["max"] else l # pyright: ignore return {} diff --git a/mpyc-web-py/lib/webloop_pyodide_old.py b/mpyc-web-py/lib/webloop_pyodide_old.py index ae83be3e..2e2f90b0 100644 --- a/mpyc-web-py/lib/webloop_pyodide_old.py +++ b/mpyc-web-py/lib/webloop_pyodide_old.py @@ -20,17 +20,17 @@ * fix run_until_complete """ -import heapq import asyncio +import contextvars +import heapq import time import traceback -import contextvars +from typing import Awaitable, Callable, Dict, Optional, Tuple import js -from typing import Dict, Tuple, Optional, Awaitable, Callable -class WebLoop(asyncio.AbstractEventLoop): +class WebLooper(asyncio.AbstractEventLoop): """A custom event loop for running asyncio in Pyodide It works by utilizing the browser event loop via the setTimeout function @@ -380,14 +380,14 @@ def get_event_loop(self): Get the current event loop """ if self._default_loop is None: - self._default_loop = WebLoop() + self._default_loop = WebLooper() return self._default_loop def new_event_loop(self): """ Create a new event loop """ - self._default_loop = WebLoop() + self._default_loop = WebLooper() return self._default_loop def set_event_loop(self, loop: asyncio.AbstractEventLoop): diff --git a/mpyc-web-py/lib/weblooper.py b/mpyc-web-py/lib/weblooper.py index d2662cb5..16b6ac85 100644 --- a/mpyc-web-py/lib/weblooper.py +++ b/mpyc-web-py/lib/weblooper.py @@ -1,2 +1,4 @@ # import .weblooperV2 from .weblooperV10 import * + +# from .webloop_pyodide_old import * diff --git a/mpyc-web-py/lib/weblooperV10.py b/mpyc-web-py/lib/weblooperV10.py index 7a0f16d5..0a1d41de 100644 --- a/mpyc-web-py/lib/weblooperV10.py +++ b/mpyc-web-py/lib/weblooperV10.py @@ -2,12 +2,13 @@ import collections import contextvars import types +from random import sample, shuffle from typing import Any, Callable, Optional import js import rich from lib.api import async_proxy -from lib.stats import stats +from lib.stats import MovingAverage, stats from pyodide.code import run_js from pyodide.ffi import IN_BROWSER, create_once_callable, create_proxy from pyodide.webloop import PyodideFuture, PyodideTask, WebLoop @@ -83,22 +84,48 @@ def trigger_run_once(self): stats_add("loop_iters") self.chan.port2.postMessage(None) + @stats.acc(lambda *args, **kwargs: stats.time()) def _run_once(self, *args, **kwargs): ntodo = len(self._ready) stats_set("ntodo", ntodo) + ntodoz(ntodo) + async_proxy.maybe_send_stats() + + # sample_indices = set(sample(range(ntodo), ntodo)) + + # ready_shuff = [self._ready[i] for i in sample_indices] + # self._ready.clear() + + # for h in ready_shuff: + # stats_add("loop_inner_iters") + # h() - for _ in range(ntodo): + while self._ready: stats_add("loop_inner_iters") self._ready.popleft()() nleft = len(self._ready) - async_proxy.maybe_send_stats() + ntodoz(nleft, "left") if nleft == 0: self.running = False else: self.trigger_run_once() +def ntodoz(ntodo, key="ready"): + if key not in stats.stats: + stats.stats[key] = { # pyright: ignore + "min_cnt": 0, + "max": 0, + "avg": MovingAverage(maxlen=200), + } + stats.stats[key]["avg"].append(ntodo) + if ntodo <= 0: + stats.stats[key]["min_cnt"] += 1 + + stats.stats[key]["max"] = max(ntodo, stats.stats[key]["max"]) if stats.stats[key]["max"] else ntodo # pyright: ignore + + def stats_add(path: str, value=1, prefix="asyncio."): stats.acc_path(f"{prefix}{path}", value) From d25b07dc6bdd4d32eb0e85d66d15dfc286dbb7ad Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Mon, 5 Feb 2024 00:00:43 +0100 Subject: [PATCH 2/8] stats and benches --- .env | 12 - .envrc | 17 +- .gitignore | 1 + benches/bnnmnist.txt | 54 +++-- benches/cnnmnist.txt | 151 +++++++----- benches/np_bnnmnist.txt | 33 +++ benches/np_cnnmnist.txt | 34 +++ benches/prof/cnnmnist.txt | 54 +++++ benches/prof/cnnmnist2.txt | 55 +++++ benches/prof/cnnmnist3.txt | 280 +++++++++++++++++++++ benches/secretsanta.txt | 52 ++-- flake.lock | 42 ++-- justfile | 5 +- mpyc-web-core/lib/transports/PeerJS.ts | 28 +-- mpyc-web-demo/index.html | 1 + mpyc-web-py/benches/profile.py | 13 + mpyc-web-py/benches/stats.py | 18 +- mpyc-web-py/lib/api/proxy.py | 26 +- mpyc-web-py/lib/api/run.py | 3 + mpyc-web-py/lib/rstats/rstats/__init__.py | 5 +- mpyc-web-py/lib/rstats/rstats/prof.py | 36 +++ mpyc-web-py/lib/rstats/rstats/stats.py | 191 +++++++++++---- mpyc-web-py/lib/stats.py | 53 ++-- mpyc-web-py/lib/weblooper.py | 3 +- mpyc-web-py/lib/weblooperBench.1.py | 142 +++++++++++ mpyc-web-py/lib/weblooperBench.py | 99 ++++++++ mpyc-web-py/lib/weblooperV10.py | 25 +- mpyc-web-py/lib/weblooperV11.py | 91 +++++++ mpyc-web-py/mpyc/mpyc/finfields.py | 281 +++++++++++----------- mpyc-web-py/mpycweb/proxy.py | 11 +- mpyc-web-starter/index.html | 4 +- nix/devenv.nix | 1 - pyproject.toml | 2 +- 33 files changed, 1403 insertions(+), 420 deletions(-) create mode 100755 benches/np_bnnmnist.txt create mode 100755 benches/np_cnnmnist.txt create mode 100755 benches/prof/cnnmnist.txt create mode 100755 benches/prof/cnnmnist2.txt create mode 100755 benches/prof/cnnmnist3.txt create mode 100644 mpyc-web-py/benches/profile.py create mode 100644 mpyc-web-py/lib/rstats/rstats/prof.py create mode 100644 mpyc-web-py/lib/weblooperBench.1.py create mode 100644 mpyc-web-py/lib/weblooperBench.py create mode 100644 mpyc-web-py/lib/weblooperV11.py diff --git a/.env b/.env index 0d8cd301..999265d0 100644 --- a/.env +++ b/.env @@ -1,13 +1 @@ # complete -o nospace -C /nix/store/zbzv9x7y2z690pj4kwpq1f4lak47w9f9-terraform-1.3.3/bin/terraform terraform - -TAILSCALE_API_KEY=op://personal/mpyc-demo/tailscale-api-token -TAILSCALE_TAILNET=op://personal/mpyc-demo/tailscale-tailnet -DIGITALOCEAN_TOKEN=op://personal/mpyc-demo/digitalocean-api-token -AWS_ACCESS_KEY_ID=op://personal/mpyc-demo/digitalocean-spaces-key-id -AWS_SECRET_ACCESS_KEY=op://personal/mpyc-demo/digitalocean-spaces-secret-access-key -SPACES_ACCESS_KEY_ID=op://personal/mpyc-demo/digitalocean-spaces-key-id -SPACES_SECRET_ACCESS_KEY=op://personal/mpyc-demo/digitalocean-spaces-secret-access-key -TF_VAR_DOMAIN=demo.mpyc.tech -G=test -VITE_HTTPS_KEY=op://personal/mpyc-demo/tls-home-nix-key -VITE_HTTPS_CERT=op://personal/mpyc-demo/tls-home-nix-crt diff --git a/.envrc b/.envrc index 8e91e745..63110da4 100644 --- a/.envrc +++ b/.envrc @@ -1,17 +1,16 @@ export DIRENV_WARN_TIMEOUT=1m -if ! has nix_direnv_version || ! nix_direnv_version 3.0.0; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.0/direnvrc" "sha256-21TMnI2xWX7HkSTjFFri2UaohXVj854mgvWapWrxRXg=" +dotenv_if_exists + +# use oprc + +if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" fi watch_file ./nix/devenv.nix -watch_file devenv.lock -watch_file devenv.yaml +nix_direnv_manual_reload + if ! use flake . --impure; then echo "devenv could not be built. The devenv environment was not loaded. Make the necessary changes to devenv.nix and hit enter to try again." >&2 fi - -if [ -f .env ]; then - dotenv - watch_file .env -fi diff --git a/.gitignore b/.gitignore index 07c0de6a..a66d6744 100644 --- a/.gitignore +++ b/.gitignore @@ -412,6 +412,7 @@ celerybeat.pid *.sage.py # Environments +.oprc .env .venv env/ diff --git a/benches/bnnmnist.txt b/benches/bnnmnist.txt index ca6fe725..c9ee9ee0 100755 --- a/benches/bnnmnist.txt +++ b/benches/bnnmnist.txt @@ -1,18 +1,32 @@ Python 3.11 +╭────────────────────────────── stats ───────────────────────────────╮ +│ asyncio: tasks: 2 / 1 / 8 k / 29 k | loop: 599 / 152 k / 152 k │ +│ data s/r: 4.6 MiB / 4.6 MiB │ +│ latency: 2.00 ms / 745 ms / 1.81 s │ +│ messages s/r: 41 k / 41 k │ +│ on_runtime_message │ +│ └── time: 4.00 μs / 51.6 μs / 353 ms │ +│ send_runtime_message │ +│ └── time: 14.0 μs / 52.4 μs / 35.1 ms │ +╰───────────────────────────── 00:01:14 ─────────────────────────────╯ + ############################# bnnmnist/3 ########################## mswsl1/native 00:27 -mswin1/web/v0.6.0-c8b8d89/edge/debug 02:25 -mswin1/web/v0.5.0-c8b8d89/edge/debug 01:32 -mswin1/web/v0.4.0-76ca881/edge/debug 01:32 +mswin1/web/v0.9.2-312c118(wl10)/edge/stats 01:14 +mswin1/web/v0.9.2-312c118(wl10)/edge 01:14 + +mswin1/web/v0.6.0-c8b8d89/edge/stats 02:25 +mswin1/web/v0.5.0-c8b8d89/edge/stats 01:32 +mswin1/web/v0.4.0-76ca881/edge/stats 01:32 mswin1/web/v0.2.1-a51a93f/edge 02:10 -mswin1/web/v0.6.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? +mswin1/web/v0.6.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? mswin1/web/v0.2.1-a51a93f/ffox ??:?? @@ -20,26 +34,30 @@ mswin1/web/v0.2.1-a51a93f/ffox ??:?? mswsl1/native 00:21 -mswin1/web/v0.6.0-c8b8d89/edge/debug 00:55 -mswin1/web/v0.5.0-76ca881/edge/debug 00:58 -mswin1/web/v0.4.0-76ca881/edge/debug 00:58 +mswin1/web/v0.9.2-312c118(wl10)/edge 00:44 + +mswin1/web/v0.6.0-c8b8d89/edge/stats 00:55 +mswin1/web/v0.5.0-76ca881/edge/stats 00:58 +mswin1/web/v0.4.0-76ca881/edge/stats 00:58 mswin1/web/v0.2.1-a51a93f/edge 01:57 / 02:00 -mswin1/web/v0.6.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? +mswin1/web/v0.6.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? mswin1/web/v0.2.1-a51a93f/ffox ??:?? ############################# bnnmnist/0 ############################# mswsl1/native 00:13 -mswin1/web/v0.6.0-c8b8d89/edge/debug 00:37 -mswin1/web/v0.5.0-c8b8d89/edge/debug 00:39 -mswin1/web/v0.4.0-76ca881/edge/debug 00:39 +mswin1/web/v0.9.2-312c118(wl10)/edge 00:32 + +mswin1/web/v0.6.0-c8b8d89/edge/stats 00:37 +mswin1/web/v0.5.0-c8b8d89/edge/stats 00:39 +mswin1/web/v0.4.0-76ca881/edge/stats 00:39 mswin1/web/v0.2.1-a51a93f/edge 00:39 -mswin1/web/v0.6.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? +mswin1/web/v0.6.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? mswin1/web/v0.2.1-a51a93f/ffox ??:?? diff --git a/benches/cnnmnist.txt b/benches/cnnmnist.txt index 0220f017..b64756f2 100755 --- a/benches/cnnmnist.txt +++ b/benches/cnnmnist.txt @@ -1,5 +1,35 @@ Python 3.11 +╭────────────────────────────── stats ──────────────────────────────╮ +│ asyncio: tasks: 2 / 1 / 113 k / 728 k | loop: 2 k / 3 M / 3 M │ +│ data s/r: 37.6 MiB / 37.6 MiB │ +│ latency: 1.00 ms / 72.7 ms / 9.18 s │ +│ messages s/r: 738 k / 738 k │ +│ on_runtime_message │ +│ └── time: 4.00 μs / 41.2 μs / 1.73 s │ +│ send_runtime_message │ +│ └── time: 14.0 μs / 42.6 μs / 81.1 ms │ +╰──────────────────────────── 00:09:30 ─────────────────────────────╯ + +╭───────────────────────────────────── stats ─────────────────────────────────────╮ +│ $time: │ +│ _run_once: ∧ 35.0 μs / μ 2.46 ms / ∨ 20.5 s / ∑ 658 s │ +│ call_later: ∧ 4.00 μs / μ 101 μs / ∨ 3.31 s / ∑ 142 s │ +│ call_later:89: ∧ 0.00 s / μ 3.45 μs / ∨ 12.6 ms / ∑ 9.23 s │ +│ nothing: ∧ 0.00 s / μ 2.94 μs / ∨ 16.9 ms / ∑ 6.70 s │ +│ nothing2: ∧ 0.00 s / μ 15.7 μs / ∨ 887 ms / ∑ 34.9 s │ +│ nothing3: ∧ 4.00 μs / μ 24.6 μs / ∨ 2.99 s / ∑ 65.6 s │ +│ on_runtime_message: ∧ 0.00 s / μ 80.3 μs / ∨ 3.31 s / ∑ 65.6 s │ +│ run_handle: ∧ 0.00 s / μ 2.00 ms / ∨ 20.5 s / ∑ 561 s │ +│ run_handle_outer1: ∧ 0.00 s / μ 2.01 ms / ∨ 20.5 s / ∑ 585 s │ +│ run_handle_outer2: ∧ 4.00 μs / μ 2.02 ms / ∨ 20.5 s / ∑ 609 s │ +│ send_runtime_message: ∧ 19.0 μs / μ 61.6 μs / ∨ 47.0 ms / ∑ 43.8 s │ +│ asyncio: tasks: 1.00 / ∨ 107 k / ∑ 728 k | loop: o 1.77 k / i 3.61 M / 2.00 │ +│ data: ⬆37.6 MiB / ⬇37.6 MiB │ +│ latency: ∧ 1.00 ms / μ 6.77 ms / ∨ 10.9 s / ∑ 1.46 Ms │ +│ messages: ⬆738 k / ⬇738 k │ +╰─────────────────────────────────── 00:14:55 ────────────────────────────────────╯ + ############################# cnnmnist/3 ########################## @@ -7,57 +37,62 @@ mswsl1/native 02:25 mswin1/native 02:19 linux2/native 02:02 -mswin1/web/v0.9.2-312c118(wl7)/chrm/debug 09:30 -mswin1/web/v0.9.2-312c118(wl6)/chrm 08:40 +mswin1/web/v0.9.2-312c118(wl10)/chrm/stats 09:30 +mswin1/web/v0.9.2-312c118(wl10)/chrm 07:43 +mswin1/web/v0.9.2-312c118(wl10)/edge/stats 09:30 +mswin1/web/v0.9.2-312c118(wl10)/edge 07:43 + +mswin1/web/v0.9.2-312c118(wl7)/chrm/stats 09:30 +mswin1/web/v0.9.2-312c118(wl7)/chrm 08:40 -mswin1/web/v0.9.2-312c118(wl6)/chrm/debug 09:46 +mswin1/web/v0.9.2-312c118(wl6)/chrm/stats 09:46 mswin1/web/v0.9.2-312c118(wl6)/chrm 08:40 -mswin1/web/v0.9.2-312c118(wl5)/chrm/debug 09:48 +mswin1/web/v0.9.2-312c118(wl5)/chrm/stats 09:48 mswin1/web/v0.9.2-312c118(wl5)/chrm 08:13 -mswin1/web/v0.9.2-312c118(wl4)/chrm/debug 09:54 +mswin1/web/v0.9.2-312c118(wl4)/chrm/stats 09:54 mswin1/web/v0.9.2-312c118(wl4)/chrm 08:21 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 10:01 +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 10:01 mswin1/web/v0.9.2-312c118(wl3)/chrm 08:40 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 09:50 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 09:50 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 09:17 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 10:29 / 10:24 / 10:14 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 10:29 / 10:24 / 10:14 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 08:46 / 12:00 / 09:06 / 09:18 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug xx:xx - queueMicrotask deadlock after Barrier 1 0 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm xx:xx - queueMicrotask deadlock after Barrier 1 0, +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats xx:xx - queueMicrotask deadlock after Barrier 1 0 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm xx:xx - queueMicrotask deadlock after Barrier 1 0 mswin1/web/v0.7.0-d09e338/edge/full_async 30:00 mswin1/web/v0.7.0-d09e338/edge/full_call_soon 24:19 mswin1/web/v0.7.0-d09e338/edge 11:37 -mswin1/web/v0.7.0-d09e338/edge/debug 11:59 -mswin1/web/v0.5.0-d09e338/edge/debug 11:32 / 11:55 -mswin1/web/v0.5.0-c8b8d89/edge/debug/console 25:51 -mswin1/web/v0.4.0-76ca881/edge/debug 12:40 / 12:45 -mswin1/web/v0.2.1-a51a93f/edge/debug 19:53 / 20:08 +mswin1/web/v0.7.0-d09e338/edge/stats 11:59 +mswin1/web/v0.5.0-d09e338/edge/stats 11:32 / 11:55 +mswin1/web/v0.5.0-c8b8d89/edge/stats/console 25:51 +mswin1/web/v0.4.0-76ca881/edge/stats 12:40 / 12:45 +mswin1/web/v0.2.1-a51a93f/edge/stats 19:53 / 20:08 mswin1/web/v0.7.0-d09e338/ffox 12:17 -mswin1/web/v0.7.0-d09e338/ffox/debug 13:16 -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? -mswin1/web/v0.2.1-a51a93f/ffox/debug ??:?? +mswin1/web/v0.7.0-d09e338/ffox/stats 13:16 +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? +mswin1/web/v0.2.1-a51a93f/ffox/stats ??:?? linux2/web/v0.7.0-d09e338/edge 13:15 -linux2/web/v0.7.0-d09e338/edge/debug 13:51 / 14:12 / 13:54 -linux2/web/v0.5.0-c8b8d89/edge/debug 14 -linux2/web/v0.4.0-76ca881/edge/debug ??:?? -linux2/web/v0.2.1-a51a93f/edge/debug ??:?? +linux2/web/v0.7.0-d09e338/edge/stats 13:51 / 14:12 / 13:54 +linux2/web/v0.5.0-c8b8d89/edge/stats 14 +linux2/web/v0.4.0-76ca881/edge/stats ??:?? +linux2/web/v0.2.1-a51a93f/edge/stats ??:?? -linux2/web/v0.7.0-d09e338/ffox/debug/full_async 36:00 -linux2/web/v0.7.0-d09e338/ffox/debug 16:00 / 16:59 -linux2/web/v0.5.0-c8b8d89/ffox/debug 19:44 / 20:42 / 20:49 -linux2/web/v0.5.0-c8b8d89/ffox/debug/10/100 36:00 -linux2/web/v0.4.0-76ca881/ffox/debug 23:38 -linux2/web/v0.2.1-a51a93f/ffox/debug ??:?? +linux2/web/v0.7.0-d09e338/ffox/stats/full_async 36:00 +linux2/web/v0.7.0-d09e338/ffox/stats 16:00 / 16:59 +linux2/web/v0.5.0-c8b8d89/ffox/stats 19:44 / 20:42 / 20:49 +linux2/web/v0.5.0-c8b8d89/ffox/stats/10/100 36:00 +linux2/web/v0.4.0-76ca881/ffox/stats 23:38 +linux2/web/v0.2.1-a51a93f/ffox/stats ??:?? mswin1/web/v0.5.0-c8b8d89/edge/dev 10:22 @@ -70,32 +105,34 @@ mswin1/native 01:20 linux2/native 01:06 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 04:08 +mswin1/web/v0.9.2-312c118(wl10)/edge 03:05 + +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 04:08 mswin1/web/v0.9.2-312c118(wl3)/chrm 03:56 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 03:36 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 03:36 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 03:38 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 03:40 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 03:40 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 03:47 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug 03:38 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats 03:38 mswin1/web/v0.9.2-03f1a8a(owl)/chrm 03:43 -mswin1/web/v0.5.0-c8b8d89/edge/debug 05:17 -mswin1/web/v0.4.0-76ca881/edge/debug 05:18 +mswin1/web/v0.5.0-c8b8d89/edge/stats 05:17 +mswin1/web/v0.4.0-76ca881/edge/stats 05:18 mswin1/web/v0.2.1-a51a93f/edge 05:57 -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? -mswin1/web/v0.2.1-a51a93f/ffox/debug ??:?? +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? +mswin1/web/v0.2.1-a51a93f/ffox/stats ??:?? -linux2/web/v0.5.0-c8b8d89/edge/debug 05:20 -linux2/web/v0.4.0-76ca881/edge/debug 05:34 +linux2/web/v0.5.0-c8b8d89/edge/stats 05:20 +linux2/web/v0.4.0-76ca881/edge/stats 05:34 linux2/web/v0.2.1-a51a93f/edge 06:13 -linux2/web/v0.5.0-c8b8d89/ffox/debug 06:02 / 05:52 / 05:46 -linux2/web/v0.4.0-76ca881/ffox/debug 06:24 / 06:17 +linux2/web/v0.5.0-c8b8d89/ffox/stats 06:02 / 05:52 / 05:46 +linux2/web/v0.4.0-76ca881/ffox/stats 06:24 / 06:17 linux2/web/v0.2.1-a51a93f/ffox 06:04 @@ -107,38 +144,40 @@ mswin1/web/v0.5.0-c8b8d89/edge/desktop-win 05:15 mswin1/web/v0.5.0-c8b8d89/edge/desktop-win/dev 05:11 -############################# cnnmnist/1/no async ############################# +############################# cnnmnist/0 ############################# mswsl1/native 00:26 mswin1/native 00:27 linux2/native 00:24 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 01:38 +mswin1/web/v0.9.2-312c118(wl10)/edge 01:17 + +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 01:38 mswin1/web/v0.9.2-312c118(wl3)/chrm 01:39 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 01:38 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 01:38 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 01:36 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 01:36 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 01:36 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 01:36 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug 01:43 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats 01:43 mswin1/web/v0.9.2-03f1a8a(owl)/chrm 01:45 -mswin1/web/v0.5.0-c8b8d89/edge/debug 01:35 -mswin1/web/v0.4.0-76ca881/edge/debug 01:38 +mswin1/web/v0.5.0-c8b8d89/edge/stats 01:35 +mswin1/web/v0.4.0-76ca881/edge/stats 01:38 mswin1/web/v0.2.1-a51a93f/edge 01:33 -mswin1/web/v0.5.0-c8b8d89/ffox/debug ??:?? -mswin1/web/v0.4.0-76ca881/ffox/debug ??:?? -mswin1/web/v0.2.1-a51a93f/ffox/debug ??:?? +mswin1/web/v0.5.0-c8b8d89/ffox/stats ??:?? +mswin1/web/v0.4.0-76ca881/ffox/stats ??:?? +mswin1/web/v0.2.1-a51a93f/ffox/stats ??:?? -linux2/web/v0.5.0-c8b8d89/edge/debug ????? -linux2/web/v0.4.0-76ca881/edge/debug 01:43 +linux2/web/v0.5.0-c8b8d89/edge/stats ????? +linux2/web/v0.4.0-76ca881/edge/stats 01:43 linux2/web/v0.2.1-a51a93f/edge ????? -linux2/web/v0.5.0-c8b8d89/ffox/debug 01:34 / 01:20 -linux2/web/v0.4.0-76ca881/ffox/debug 01:41 / 01:21 +linux2/web/v0.5.0-c8b8d89/ffox/stats 01:34 / 01:20 +linux2/web/v0.4.0-76ca881/ffox/stats 01:41 / 01:21 linux2/web/v0.2.1-a51a93f/ffox 01:21 / 01:21 diff --git a/benches/np_bnnmnist.txt b/benches/np_bnnmnist.txt new file mode 100755 index 00000000..0f8ce49e --- /dev/null +++ b/benches/np_bnnmnist.txt @@ -0,0 +1,33 @@ +Python 3.11 + +╭─────────────────────────── stats ────────────────────────────╮ +│ asyncio: tasks: 2 / 1 / 52 / 221 | loop: 608 / 1 k / 1 k │ +│ data s/r: 4.8 MiB / 4.8 MiB │ +│ latency: 1.00 ms / 147 ms / 650 ms │ +│ messages s/r: 113 / 109 │ +│ on_runtime_message │ +│ └── time: 10.0 μs / 97.3 μs / 485 μs │ +│ send_runtime_message │ +│ └── time: 19.0 μs / 91.8 μs / 539 μs │ +╰────────────────────────── 00:00:22 ──────────────────────────╯ + + +############################# np_bnnmnist/3 ########################## + +mswsl1/native 00:15 + +mswin1/web/v0.9.2-312c118(wl10)/edge/stats 00:22 +mswin1/web/v0.9.2-312c118(wl10)/edge 00:22 + + +############################# np_bnnmnist/1 ############################# + +mswsl1/native 00:13 + +mswin1/web/v0.9.2-312c118(wl10)/edge 00:15 + +############################# np_bnnmnist/0 ############################# + +mswsl1/native 00:08 + +mswin1/web/v0.9.2-312c118(wl10)/edge 00:15 diff --git a/benches/np_cnnmnist.txt b/benches/np_cnnmnist.txt new file mode 100755 index 00000000..2f311123 --- /dev/null +++ b/benches/np_cnnmnist.txt @@ -0,0 +1,34 @@ +Python 3.11 + +╭─────────────────────────── stats ────────────────────────────╮ +│ asyncio: tasks: 2 / 1 / 61 / 488 | loop: 1 k / 3 k / 3 k │ +│ data s/r: 33.9 MiB / 33.9 MiB │ +│ latency: 1.00 ms / 821 ms / 12.2 s │ +│ messages s/r: 219 / 215 │ +│ on_runtime_message │ +│ └── time: 9.00 μs / 115 μs / 854 μs │ +│ send_runtime_message │ +│ └── time: 20.0 μs / 162 μs / 2.21 ms │ +╰────────────────────────── 00:01:08 ──────────────────────────╯ + +############################# np_cnnmnist/3 ########################## + +mswsl1/native 00:14 + +mswin1/web/v0.9.2-312c118(wl10)/edge/stats 01:08 +mswin1/web/v0.9.2-312c118(wl10)/edge 01:07 + + +############################# np_cnnmnist/1 ############################# + +mswsl1/native 00:08 + +mswin1/web/v0.9.2-312c118(wl10)/edge 00:30 +mswin1/web/v0.9.2-312c118(wl10)/chrm 00:37 + +############################# np_cnnmnist/0 ############################# + +mswsl1/native 00:04 + +mswin1/web/v0.9.2-312c118(wl10)/edge 00:30 +mswin1/web/v0.9.2-312c118(wl10)/chrm 00:37 diff --git a/benches/prof/cnnmnist.txt b/benches/prof/cnnmnist.txt new file mode 100755 index 00000000..8259786b --- /dev/null +++ b/benches/prof/cnnmnist.txt @@ -0,0 +1,54 @@ + 242909252 function calls (226538376 primitive calls) in 598.324 seconds + + Ordered by: internal time + List reduced from 860 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) +5434735/2062 52.003 0.000 598.338 0.290 contextlib.py:78(inner) + 9573824 39.498 0.000 50.635 0.000 finfields.py:373(__init__) +45870687/45870684 25.529 0.000 25.542 0.000 {built-in method builtins.isinstance} + 5434735 25.002 0.000 81.088 0.000 stats.py:386(__exit__) + 2716846 24.035 0.000 33.155 0.000 weblooperBench.py:127(make_handle) + 759710 23.584 0.000 23.584 0.000 {built-in method gmpy2.gmpy2.powmod} + 5434735 22.627 0.000 33.147 0.000 stats.py:69(append) + 5434735 17.485 0.000 50.632 0.000 stats.py:92(update) + 3445574 16.811 0.000 25.458 0.000 stats.py:141(acc_path) + 5434735 15.075 0.000 20.808 0.000 stats.py:368(__enter__) + 1043 14.969 0.014 598.244 0.574 weblooperBench.py:109(_run_once) +4928405/302832 14.902 0.000 21.758 0.000 asyncoro.py:192(_add_callbacks) + 4021753 14.336 0.000 37.949 0.000 sectypes.py:516(__init__) +4928405/302832 14.188 0.000 20.392 0.000 asyncoro.py:208(_get_results) + 2716337 12.566 0.000 216.722 0.000 weblooperBench.py:67(call_soon) + 728194 12.549 0.000 158.014 0.000 asyncoro.py:396(typed_asyncoro) + 10869470 11.186 0.000 11.186 0.000 {built-in method time.perf_counter_ns} + 2080 10.085 0.005 10.114 0.005 saved.py:86(inprod2D) + 2716846 9.156 0.000 139.379 0.000 weblooperBench.py:78(call_later) + 3275427 8.614 0.000 43.522 0.000 saved.py:27() + + + 242909252 function calls (226538376 primitive calls) in 598.324 seconds + + Ordered by: cumulative time + List reduced from 860 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) +5434735/2062 52.003 0.000 598.338 0.290 contextlib.py:78(inner) + 1043 0.018 0.000 598.296 0.574 weblooperBench.py:58(run_once_proxy) + 1043 14.969 0.014 598.244 0.574 weblooperBench.py:109(_run_once) + 2716846 4.212 0.000 549.703 0.000 weblooperBench.py:41(run_handle) + 2716846 6.433 0.000 545.491 0.000 events.py:78(_run) + 2716846 7.672 0.000 539.057 0.000 {method 'run' of '_contextvars.Context' objects} +1862910/1356334 4.700 0.000 289.314 0.000 {method 'send' of 'coroutine' objects} + 1173385 2.268 0.000 288.269 0.000 asyncoro.py:282(_wrap_in_coro) + 1173385 4.573 0.000 286.001 0.000 asyncoro.py:266(__await__) + 2716337 12.566 0.000 216.722 0.000 weblooperBench.py:67(call_soon) + 728194 12.549 0.000 158.014 0.000 asyncoro.py:396(typed_asyncoro) + 2716846 9.156 0.000 139.379 0.000 weblooperBench.py:78(call_later) + 728194 1.037 0.000 86.001 0.000 asyncoro.py:431() + 728194 2.245 0.000 84.963 0.000 asyncoro.py:348(_reconcile) +1263482/728194 4.301 0.000 82.213 0.000 asyncoro.py:363(__reconcile) + 1531485 3.088 0.000 82.201 0.000 {method 'set_result' of '_asyncio.Future' objects} + 257 0.563 0.002 81.418 0.317 saved.py:129(main) + 256 0.001 0.000 81.401 0.318 run.py:68(run_code) + 256 0.000 0.000 81.401 0.318 _base.py:505(eval_code_async) + 256 0.001 0.000 81.400 0.318 _base.py:350(run_async) diff --git a/benches/prof/cnnmnist2.txt b/benches/prof/cnnmnist2.txt new file mode 100755 index 00000000..28863a12 --- /dev/null +++ b/benches/prof/cnnmnist2.txt @@ -0,0 +1,55 @@ + 226378851 function calls (213513193 primitive calls) in 802.705 seconds + + Ordered by: internal time + List reduced from 843 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) + 10372252 59.115 0.000 73.189 0.000 finfields.py:373(__init__) +55979813/55979810 38.257 0.000 38.263 0.000 {built-in method builtins.isinstance} + 2716413 29.531 0.000 52.313 0.000 weblooperBench.py:128(make_handle) + 759710 26.537 0.000 26.537 0.000 {built-in method gmpy2.gmpy2.powmod} +6134367/536653 24.342 0.000 35.582 0.000 asyncoro.py:192(_add_callbacks) + 738331 22.183 0.000 22.183 0.000 proxy.py:114(postMessage) +6134367/536653 22.052 0.000 32.068 0.000 asyncoro.py:208(_get_results) + 2716413 21.055 0.000 22.782 0.000 events.py:31(__init__) + 738328 20.805 0.000 40.983 0.000 proxy.py:124(_onmessage) + 738331 18.809 0.000 18.809 0.000 {built-in method _pyodide_core.to_js} + 388496 18.806 0.000 23.758 0.000 thresha.py:88(recombine) + 2716082 17.710 0.000 95.046 0.000 weblooperBench.py:67(call_soon) + 4021753 16.069 0.000 42.346 0.000 sectypes.py:516(__init__) + 309364 15.309 0.000 97.728 0.000 runtime.py:463(output) + 728195 14.894 0.000 29.790 0.000 asyncoro.py:306(returnType) + 1095 14.836 0.014 758.991 0.693 weblooperBench.py:110(_run_once) + 464016 14.401 0.000 26.056 0.000 thresha.py:238(__call__) + 728195 13.459 0.000 122.999 0.000 asyncoro.py:396(typed_asyncoro) + 468585 11.726 0.000 187.761 0.000 runtime.py:555(_reshare) + 2716413 11.409 0.000 77.373 0.000 weblooperBench.py:79(call_later) + + + 226378851 function calls (213513193 primitive calls) in 802.705 seconds + + Ordered by: cumulative time + List reduced from 843 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1095 0.018 0.000 759.009 0.693 weblooperBench.py:58(run_once_proxy) + 1095 14.836 0.014 758.991 0.693 weblooperBench.py:110(_run_once) + 2716413 5.059 0.000 739.429 0.000 weblooperBench.py:41(run_handle) + 2716413 7.844 0.000 734.369 0.000 events.py:78(_run) + 2716413 10.766 0.000 726.526 0.000 {method 'run' of '_contextvars.Context' objects} + 1173387 3.322 0.000 553.010 0.000 asyncoro.py:282(_wrap_in_coro) +1862912/1356336 6.336 0.000 550.946 0.000 {method 'send' of 'coroutine' objects} + 1173387 7.495 0.000 549.688 0.000 asyncoro.py:266(__await__) + 468585 11.726 0.000 187.761 0.000 runtime.py:555(_reshare) + 728195 13.459 0.000 122.999 0.000 asyncoro.py:396(typed_asyncoro) + 232008 8.299 0.000 104.019 0.000 runtime.py:3526(random_bits) + 309364 15.309 0.000 97.728 0.000 runtime.py:463(output) + 2716082 17.710 0.000 95.046 0.000 weblooperBench.py:67(call_soon) + 613990 6.375 0.000 78.414 0.000 asyncoro.py:224(gather_shares) + 2716413 11.409 0.000 77.373 0.000 weblooperBench.py:79(call_later) + 237 0.006 0.000 76.777 0.324 saved.py:5(main) + 237 0.549 0.002 76.771 0.324 cnnmnist.py:129(main) + 236 0.001 0.000 76.705 0.325 run.py:68(run_code) + 236 0.000 0.000 76.705 0.325 _base.py:505(eval_code_async) + 236 0.001 0.000 76.704 0.325 _base.py:350(run_async) + diff --git a/benches/prof/cnnmnist3.txt b/benches/prof/cnnmnist3.txt new file mode 100755 index 00000000..506d5ec9 --- /dev/null +++ b/benches/prof/cnnmnist3.txt @@ -0,0 +1,280 @@ + 206404632 function calls (193538974 primitive calls) in 753.123 seconds + + Ordered by: internal time + List reduced from 825 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) + 10372252 49.242 0.000 57.314 0.000 /lib/python3.11/site-packages/mpyc/finfields.py:374(__init__) + 738326 39.418 0.000 56.506 0.000 /lib/python3.11/site-packages/lib/api/proxy.py:91(send) +45605393/45605390 32.573 0.000 32.578 0.000 {built-in method builtins.isinstance} + 2718168 27.701 0.000 71.431 0.000 /lib/python3.11/site-packages/lib/weblooperBench.py:55(call_soon) + 759710 26.541 0.000 26.541 0.000 {built-in method gmpy2.gmpy2.powmod} + 738322 26.089 0.000 45.604 0.000 /lib/python3.11/site-packages/lib/api/proxy.py:122(onmessage) +6134367/536653 23.573 0.000 35.121 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:192(_add_callbacks) + 2718483 23.385 0.000 43.765 0.000 /lib/python3.11/site-packages/lib/weblooperBench.py:67(call_later) +6134367/536653 22.315 0.000 32.399 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:208(_get_results) + 388496 18.523 0.000 23.377 0.000 /lib/python3.11/site-packages/mpyc/thresha.py:88(recombine) + 4021753 16.542 0.000 37.835 0.000 /lib/python3.11/site-packages/mpyc/sectypes.py:516(__init__) + 2718483 16.495 0.000 18.135 0.000 /lib/python311.zip/asyncio/events.py:31(__init__) + 738326 15.369 0.000 15.369 0.000 {built-in method _pyodide_core.to_js} + 728195 13.841 0.000 106.823 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) + 464016 13.296 0.000 25.636 0.000 /lib/python3.11/site-packages/mpyc/thresha.py:238(__call__) + 1117 12.309 0.011 707.667 0.634 /lib/python3.11/site-packages/lib/weblooperBench.py:95(_run_once) + 468799 11.329 0.000 190.989 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) + 310208 11.074 0.000 88.008 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:463(output) + 2080 10.808 0.005 10.838 0.005 /lib/python3.11/site-packages/cnnmnist.py:86(inprod2D) + 2718483 10.520 0.000 683.572 0.000 {method 'run' of '_contextvars.Context' objects} + + + Ordered by: internal time + List reduced from 825 to 20 due to restriction <20> + +Function was called by... + ncalls tottime cumtime +/lib/python3.11/site-packages/mpyc/finfields.py:374(__init__) <- 571785 1.487 1.869 /lib/python3.11/site-packages/mpyc/finfields.py:101(__add__) + 77345 0.248 0.312 /lib/python3.11/site-packages/mpyc/finfields.py:129(__sub__) + 38668 0.125 0.156 /lib/python3.11/site-packages/mpyc/finfields.py:139(__rsub__) + 875735 4.313 4.956 /lib/python3.11/site-packages/mpyc/finfields.py:165(__mul__) + 837057 4.925 5.756 /lib/python3.11/site-packages/mpyc/runtime.py:540() + 798385 5.901 6.877 /lib/python3.11/site-packages/mpyc/runtime.py:616() + 759710 7.286 7.936 /lib/python3.11/site-packages/mpyc/runtime.py:1339(sgn) + 1034 0.033 0.035 /lib/python3.11/site-packages/mpyc/runtime.py:2241(matrix_prod) + 759710 2.060 2.627 /lib/python3.11/site-packages/mpyc/runtime.py:3526(random_bits) + 759710 4.947 5.485 /lib/python3.11/site-packages/mpyc/runtime.py:3585() + 3296316 12.736 14.892 /lib/python3.11/site-packages/mpyc/sectypes.py:516(__init__) + 29 0.000 0.000 +/lib/python3.11/site-packages/mpyc/thresha.py:67(_recombination_vector) + 12 0.000 0.000 /lib/python3.11/site-packages/mpyc/thresha.py:74() + 837046 2.427 3.096 /lib/python3.11/site-packages/mpyc/thresha.py:144(pseudorandom_share) + 759710 2.754 3.317 +/lib/python3.11/site-packages/mpyc/thresha.py:176(pseudorandom_share_zero) +/lib/python3.11/site-packages/lib/api/proxy.py:91(send) <- 2 0.000 0.000 /lib/python3.11/site-packages/mpycweb/proxy.py:81(send_ready_message) + 738324 39.418 56.506 +/lib/python3.11/site-packages/mpycweb/proxy.py:103(send_runtime_message) +{built-in method builtins.isinstance} <- 8336 0.005 0.005 /lib/python3.11/site-packages/cnnmnist.py:41(dim) + 14324879 9.837 9.837 /lib/python3.11/site-packages/mpyc/asyncoro.py:192(_add_callbacks) + 14324879 9.694 9.694 /lib/python3.11/site-packages/mpyc/asyncoro.py:208(_get_results) + 1841968 1.940 1.940 /lib/python3.11/site-packages/mpyc/asyncoro.py:224(gather_shares) + 1533726 1.198 1.198 /lib/python3.11/site-packages/mpyc/asyncoro.py:306(returnType) + 2526966 1.746 1.746 /lib/python3.11/site-packages/mpyc/asyncoro.py:363(__reconcile) + 1263483 1.003 1.003 /lib/python3.11/site-packages/mpyc/asyncoro.py:375(_ncopy) + 2 0.000 0.000 /lib/python3.11/site-packages/mpyc/finfields.py:23(GF) + 1114285 0.662 0.662 /lib/python3.11/site-packages/mpyc/finfields.py:101(__add__) + 77345 0.047 0.047 /lib/python3.11/site-packages/mpyc/finfields.py:129(__sub__) + 38668 0.026 0.026 /lib/python3.11/site-packages/mpyc/finfields.py:139(__rsub__) + 953083 0.555 0.555 /lib/python3.11/site-packages/mpyc/finfields.py:165(__mul__) + 72 0.000 0.000 /lib/python3.11/site-packages/mpyc/finfields.py:182(__imul__) + 77348 0.052 0.052 /lib/python3.11/site-packages/mpyc/finfields.py:193(__truediv__) + 38668 0.022 0.022 /lib/python3.11/site-packages/mpyc/finfields.py:235(__ilshift__) + 77336 0.088 0.088 /lib/python3.11/site-packages/mpyc/finfields.py:285(__eq__) + 4 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:296(transfer) + 464022 0.474 0.474 /lib/python3.11/site-packages/mpyc/runtime.py:463(output) + 467640 0.401 0.401 /lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) + 38677 0.022 0.022 /lib/python3.11/site-packages/mpyc/runtime.py:988(mul) + 10 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:1519(_argmax) + 56484 0.036 0.036 /lib/python3.11/site-packages/mpyc/runtime.py:2153(if_else) + 4 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:2241(matrix_prod) + 77336 0.068 0.068 /lib/python3.11/site-packages/mpyc/runtime.py:3434(_random) + 686769 0.418 0.418 /lib/python3.11/site-packages/mpyc/sectypes.py:46(set_share) + 211192 0.115 0.115 /lib/python3.11/site-packages/mpyc/sectypes.py:140(_coerce) + 77354 0.039 0.039 /lib/python3.11/site-packages/mpyc/sectypes.py:154(_coerce2) + 38677 0.026 0.026 /lib/python3.11/site-packages/mpyc/sectypes.py:207(__mul__) + 3412320 1.793 1.793 /lib/python3.11/site-packages/mpyc/sectypes.py:516(__init__) + 233820 0.311 0.311 /lib/python3.11/site-packages/mpyc/thresha.py:23(random_split) + 1165488 1.533 1.533 /lib/python3.11/site-packages/mpyc/thresha.py:88(recombine) + 464016 0.453 0.453 /lib/python3.11/site-packages/mpyc/thresha.py:238(__call__) + 8 0.000 0.000 /lib/python3.11/site-packages/numpy/compat/py3k.py:49(isfileobj) + 8 0.000 0.000 /lib/python3.11/site-packages/numpy/lib/format.py:282(descr_to_dtype) + 24 0.000 0.000 +/lib/python3.11/site-packages/numpy/lib/format.py:587(_read_array_header) + 16 0.000 0.000 /lib/python3.11/site-packages/numpy/lib/format.py:652() + 13 0.000 0.000 +/lib/python3.11/site-packages/numpy/lib/function_base.py:2285(__init__) + 13 0.000 0.000 +/lib/python3.11/site-packages/numpy/lib/function_base.py:2374(_get_ufunc_and_otypes) + 2691 0.002 0.002 /lib/python3.11/site-packages/rich/console.py:170(update) + 1984 0.001 0.001 /lib/python3.11/site-packages/rich/console.py:1287(render) + 1625 0.001 0.001 /lib/python3.11/site-packages/rich/console.py:1456(get_style) + 157 0.000 0.005 +/lib/python3.11/site-packages/rich/console.py:1486(_collect_renderables) + 106 0.000 0.000 /lib/python3.11/site-packages/rich/highlighter.py:20(__call__) + 49 0.000 0.000 /lib/python3.11/site-packages/rich/logging.py:126(emit) + 441 0.000 0.000 /lib/python3.11/site-packages/rich/measure.py:78(get) + 441 0.000 0.000 /lib/python3.11/site-packages/rich/padding.py:60(unpack) + 50 0.000 0.000 /lib/python3.11/site-packages/rich/pretty.py:154(_safe_isinstance) + 26 0.000 0.000 /lib/python3.11/site-packages/rich/pretty.py:542(_is_namedtuple) + 637 0.001 0.001 /lib/python3.11/site-packages/rich/protocol.py:10(is_renderable) + 34 0.000 0.000 /lib/python3.11/site-packages/rich/style.py:146(_make_color) + 108 0.000 0.000 /lib/python3.11/site-packages/rich/style.py:422(__eq__) + 49 0.000 0.000 /lib/python3.11/site-packages/rich/text.py:185(__eq__) + 229 0.000 0.000 /lib/python3.11/site-packages/rich/text.py:190(__contains__) + 104 0.000 0.000 /lib/python3.11/site-packages/rich/text.py:962(append) + 8 0.000 0.000 /lib/python311.zip/ast.py:33(parse) + 16 0.000 0.000 /lib/python311.zip/ast.py:54(literal_eval) + 120 0.000 0.000 /lib/python311.zip/ast.py:84(_convert) + 1 0.000 0.000 /lib/python311.zip/collections/__init__.py:348(namedtuple) + 13 0.000 0.000 /lib/python311.zip/dataclasses.py:1249(is_dataclass) + 8 0.000 0.000 /lib/python311.zip/enum.py:1501(__and__) + 2 0.000 0.000 /lib/python311.zip/gzip.py:25(open) + 500 0.001 0.001 /lib/python311.zip/inspect.py:292(isclass) + 2 0.000 0.000 /lib/python311.zip/logging/__init__.py:292(__init__) + 98 0.000 0.000 /lib/python311.zip/pathlib.py:484(_parse_args) + 1 0.000 0.000 /lib/python311.zip/pstats.py:137(load_stats) + 393 0.000 0.000 /lib/python311.zip/re/__init__.py:253(escape) + 273 0.001 0.001 /lib/python311.zip/re/__init__.py:272(_compile) + 4 0.000 0.000 /lib/python311.zip/re/_compiler.py:568(isstring) + 71 0.000 0.000 /lib/python311.zip/re/_parser.py:162(__getitem__) + 2 0.000 0.000 /lib/python311.zip/re/_parser.py:222(__init__) + 2 0.000 0.000 /lib/python311.zip/re/_parser.py:954(fix_flags) + 8 0.000 0.000 /lib/python311.zip/typing.py:159(_type_convert) + 15 0.000 0.000 /lib/python311.zip/typing.py:168(_type_check) + 7 0.000 0.000 /lib/python311.zip/typing.py:245(_collect_parameters) + 2 0.000 0.000 /lib/python311.zip/typing.py:308(_remove_dups_flatten) + 1 0.000 0.000 /lib/python311.zip/typing.py:645(Union) + 4 0.000 0.000 /lib/python311.zip/typing.py:1340(__init__) + 9 0.000 0.000 /lib/python311.zip/typing.py:1352(__eq__) + 3 0.000 0.000 /lib/python311.zip/typing.py:1550(__getitem__) + 24 0.000 0.000 /lib/python311.zip/typing.py:1651(__eq__) + 3 0.000 0.000 /lib/python311.zip/typing.py:2038(_proto_hook) + 2 0.000 0.000 :1127(get_data) + 1 0.000 0.000 :1464(_get_spec) + 1 0.000 0.000 :200(makedirs) + 80 0.000 0.000 :41(_get_sep) + 49 0.000 0.000 :117(splitext) +/lib/python3.11/site-packages/lib/weblooperBench.py:55(call_soon) <- 728195 18.071 41.659 /lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) + 2 0.000 0.000 /lib/python311.zip/pyodide/webloop.py:428(create_task) + 728197 3.992 13.588 {method 'run' of '_contextvars.Context' objects} + 1261774 5.638 16.184 {method 'set_result' of '_asyncio.Future' objects} +{built-in method gmpy2.gmpy2.powmod} <- 759710 26.541 26.541 /lib/python3.11/site-packages/mpyc/finfields.py:436(_sqrt) +/lib/python3.11/site-packages/lib/api/proxy.py:122(onmessage) <- +/lib/python3.11/site-packages/mpyc/asyncoro.py:192(_add_callbacks) <- 536653 4.363 35.121 /lib/python3.11/site-packages/mpyc/asyncoro.py:178(__init__) + 5597714/1302509 19.209 29.312 +/lib/python3.11/site-packages/mpyc/asyncoro.py:192(_add_callbacks) +/lib/python3.11/site-packages/lib/weblooperBench.py:67(call_later) <- 2718168 23.358 43.730 /lib/python3.11/site-packages/lib/weblooperBench.py:55(call_soon) + 315 0.027 0.035 /lib/python311.zip/asyncio/tasks.py:627(sleep) +/lib/python3.11/site-packages/mpyc/asyncoro.py:208(_get_results) <- 435423 4.545 26.157 /lib/python3.11/site-packages/mpyc/asyncoro.py:178(__init__) + 101230 0.615 6.241 /lib/python3.11/site-packages/mpyc/asyncoro.py:187(_decrement) + 5597714/1302509 17.155 25.869 /lib/python3.11/site-packages/mpyc/asyncoro.py:208(_get_results) +/lib/python3.11/site-packages/mpyc/thresha.py:88(recombine) <- 154674 7.766 9.529 /lib/python3.11/site-packages/mpyc/runtime.py:463(output) + 233820 10.757 13.848 /lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) + 2 0.000 0.000 /lib/python3.11/site-packages/mpyc/thresha.py:135(_f_S_i) +/lib/python3.11/site-packages/mpyc/sectypes.py:516(__init__) <- 3275427 12.100 30.538 /lib/python3.11/site-packages/cnnmnist.py:27() + 503828 2.428 4.201 /lib/python3.11/site-packages/mpyc/asyncoro.py:302() + 182941 1.689 2.341 /lib/python3.11/site-packages/mpyc/asyncoro.py:306(returnType) + 38668 0.238 0.371 /lib/python3.11/site-packages/mpyc/runtime.py:1339(sgn) + 10 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:1519(_argmax) + 20879 0.087 0.385 /lib/python3.11/site-packages/mpyc/sectypes.py:140(_coerce) +/lib/python311.zip/asyncio/events.py:31(__init__) <- 2718483 16.495 18.135 /lib/python3.11/site-packages/lib/weblooperBench.py:67(call_later) +{built-in method _pyodide_core.to_js} <- 738326 15.369 15.369 /lib/python3.11/site-packages/lib/api/proxy.py:91(send) +/lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) <- 3872 0.077 6.793 /lib/python3.11/site-packages/cnnmnist.py:52(convolvetensor) + 4 0.000 0.635 /lib/python3.11/site-packages/cnnmnist.py:111() + 5 0.000 0.495 /lib/python3.11/site-packages/cnnmnist.py:129(main) + 38668 0.606 5.211 /lib/python3.11/site-packages/mpyc/runtime.py:807(is_zero_public) + 38677 1.728 4.547 /lib/python3.11/site-packages/mpyc/runtime.py:988(mul) + 38668 0.589 4.237 /lib/python3.11/site-packages/mpyc/runtime.py:1281(lt) + 193340 3.995 32.194 /lib/python3.11/site-packages/mpyc/runtime.py:1339(sgn) + 193349 2.861 28.248 /lib/python3.11/site-packages/mpyc/runtime.py:1898(prod) + 2 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:2241(matrix_prod) + 77336 2.154 11.139 /lib/python3.11/site-packages/mpyc/runtime.py:3526(random_bits) + 28251 0.327 2.489 /lib/python3.11/site-packages/mpyc/sectypes.py:181(__add__) + 66910 0.826 5.847 /lib/python3.11/site-packages/mpyc/sectypes.py:191(__sub__) + 10435 0.127 1.005 /lib/python3.11/site-packages/mpyc/sectypes.py:199(__rsub__) + 38677 0.552 3.982 /lib/python3.11/site-packages/mpyc/sectypes.py:207(__mul__) + 1 0.000 0.000 /lib/python3.11/site-packages/mpycweb/patches.py:139(shutdown) +/lib/python3.11/site-packages/mpyc/thresha.py:238(__call__) <- 309344 10.865 17.744 /lib/python3.11/site-packages/mpyc/thresha.py:144(pseudorandom_share) + 154672 2.431 7.892 +/lib/python3.11/site-packages/mpyc/thresha.py:176(pseudorandom_share_zero) +/lib/python3.11/site-packages/lib/weblooperBench.py:95(_run_once) <- +/lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) <- 468799 11.329 190.989 {method 'send' of 'coroutine' objects} +/lib/python3.11/site-packages/mpyc/runtime.py:463(output) <- 310208 11.074 88.008 {method 'send' of 'coroutine' objects} +/lib/python3.11/site-packages/cnnmnist.py:86(inprod2D) <- 2080 10.808 10.838 /lib/python3.11/site-packages/cnnmnist.py:52(convolvetensor) +{method 'run' of '_contextvars.Context' objects} <- 2718483 10.520 683.572 /lib/python311.zip/asyncio/events.py:78(_run) + + + 206404632 function calls (193538974 primitive calls) in 753.123 seconds + + Ordered by: cumulative time + List reduced from 825 to 20 due to restriction <20> + + ncalls tottime percall cumtime percall filename:lineno(function) + 1117 12.309 0.011 707.667 0.634 /lib/python3.11/site-packages/lib/weblooperBench.py:95(_run_once) + 2718483 7.467 0.000 691.039 0.000 /lib/python311.zip/asyncio/events.py:78(_run) + 2718483 10.520 0.000 683.572 0.000 {method 'run' of '_contextvars.Context' objects} + 1174440 3.288 0.000 531.741 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:282(_wrap_in_coro) +1863965/1357389 9.340 0.000 530.503 0.000 {method 'send' of 'coroutine' objects} + 1174440 6.820 0.000 528.453 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:266(__await__) + 468799 11.329 0.000 190.989 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) + 728195 13.841 0.000 106.823 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) + 232008 7.562 0.000 97.058 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:3526(random_bits) + 310208 11.074 0.000 88.008 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:463(output) + 613990 6.202 0.000 77.702 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:224(gather_shares) + 738322 7.762 0.000 72.304 0.000 /lib/python3.11/site-packages/mpyc/asyncoro.py:67(send) + 2718168 27.701 0.000 71.431 0.000 /lib/python3.11/site-packages/lib/weblooperBench.py:55(call_soon) + 231 0.005 0.000 70.842 0.307 ./py/benches/profile.py:5(main) + 231 0.550 0.002 70.837 0.307 /lib/python3.11/site-packages/cnnmnist.py:129(main) + 230 0.001 0.000 70.758 0.308 /lib/python3.11/site-packages/lib/api/run.py:68(run_code) + 230 0.000 0.000 70.758 0.308 /lib/python311.zip/_pyodide/_base.py:505(eval_code_async) + 230 0.000 0.000 70.757 0.308 /lib/python311.zip/_pyodide/_base.py:350(run_async) + 230 0.001 0.000 70.757 0.308 ./py/benches/profile.py:1() + 230 0.001 0.000 70.756 0.308 /lib/python3.11/site-packages/lib/rstats/rstats/prof.py:19(_aprof) + + + Ordered by: cumulative time + List reduced from 825 to 20 due to restriction <20> + +Function was called by... + ncalls tottime cumtime +/lib/python3.11/site-packages/lib/weblooperBench.py:95(_run_once) <- +/lib/python311.zip/asyncio/events.py:78(_run) <- 315 0.001 0.014 /lib/python3.11/site-packages/lib/weblooperBench.py:88() + 2718168 7.466 691.025 /lib/python3.11/site-packages/lib/weblooperBench.py:95(_run_once) +{method 'run' of '_contextvars.Context' objects} <- 2718483 10.520 683.572 /lib/python311.zip/asyncio/events.py:78(_run) +/lib/python3.11/site-packages/mpyc/asyncoro.py:282(_wrap_in_coro) <- 1174440 3.288 531.741 {method 'run' of '_contextvars.Context' objects} +{method 'send' of 'coroutine' objects} <- 1174440 4.884 521.633 /lib/python3.11/site-packages/mpyc/asyncoro.py:266(__await__) + 689525 4.456 39.222 /lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) +/lib/python3.11/site-packages/mpyc/asyncoro.py:266(__await__) <- 1174440 6.820 528.453 /lib/python3.11/site-packages/mpyc/asyncoro.py:282(_wrap_in_coro) +/lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) <- 468799 11.329 190.989 {method 'send' of 'coroutine' objects} +/lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) <- 3872 0.077 6.793 /lib/python3.11/site-packages/cnnmnist.py:52(convolvetensor) + 4 0.000 0.635 /lib/python3.11/site-packages/cnnmnist.py:111() + 5 0.000 0.495 /lib/python3.11/site-packages/cnnmnist.py:129(main) + 38668 0.606 5.211 /lib/python3.11/site-packages/mpyc/runtime.py:807(is_zero_public) + 38677 1.728 4.547 /lib/python3.11/site-packages/mpyc/runtime.py:988(mul) + 38668 0.589 4.237 /lib/python3.11/site-packages/mpyc/runtime.py:1281(lt) + 193340 3.995 32.194 /lib/python3.11/site-packages/mpyc/runtime.py:1339(sgn) + 193349 2.861 28.248 /lib/python3.11/site-packages/mpyc/runtime.py:1898(prod) + 2 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:2241(matrix_prod) + 77336 2.154 11.139 /lib/python3.11/site-packages/mpyc/runtime.py:3526(random_bits) + 28251 0.327 2.489 /lib/python3.11/site-packages/mpyc/sectypes.py:181(__add__) + 66910 0.826 5.847 /lib/python3.11/site-packages/mpyc/sectypes.py:191(__sub__) + 10435 0.127 1.005 /lib/python3.11/site-packages/mpyc/sectypes.py:199(__rsub__) + 38677 0.552 3.982 /lib/python3.11/site-packages/mpyc/sectypes.py:207(__mul__) + 1 0.000 0.000 /lib/python3.11/site-packages/mpycweb/patches.py:139(shutdown) +/lib/python3.11/site-packages/mpyc/runtime.py:3526(random_bits) <- 232008 7.562 97.058 {method 'send' of 'coroutine' objects} +/lib/python3.11/site-packages/mpyc/runtime.py:463(output) <- 310208 11.074 88.008 {method 'send' of 'coroutine' objects} +/lib/python3.11/site-packages/mpyc/asyncoro.py:224(gather_shares) <- 6 0.000 1.331 /lib/python3.11/site-packages/cnnmnist.py:52(convolvetensor) + 2 0.000 0.000 /lib/python3.11/site-packages/mpyc/runtime.py:296(transfer) + 154676 1.673 11.859 /lib/python3.11/site-packages/mpyc/runtime.py:463(output) + 235612 3.209 25.752 /lib/python3.11/site-packages/mpyc/runtime.py:555(_reshare) + 38668 0.165 0.258 /lib/python3.11/site-packages/mpyc/runtime.py:807(is_zero_public) + 28251 0.189 1.036 /lib/python3.11/site-packages/mpyc/runtime.py:938(add) + 77345 0.526 3.771 /lib/python3.11/site-packages/mpyc/runtime.py:949(sub) + 38677 0.263 1.495 /lib/python3.11/site-packages/mpyc/runtime.py:988(mul) + 38668 0.165 0.263 /lib/python3.11/site-packages/mpyc/runtime.py:1339(sgn) + 2 0.000 0.012 /lib/python3.11/site-packages/mpyc/runtime.py:2040(vector_add) + 2080 0.014 7.719 /lib/python3.11/site-packages/mpyc/runtime.py:2082(matrix_add) + 2 0.000 24.206 /lib/python3.11/site-packages/mpyc/runtime.py:2241(matrix_prod) + 1 0.000 0.000 /lib/python3.11/site-packages/mpycweb/patches.py:139(shutdown) +/lib/python3.11/site-packages/mpyc/asyncoro.py:67(send) <- 270682 2.632 24.664 /lib/python3.11/site-packages/mpyc/runtime.py:119(_send_message) + 467640 5.130 47.640 /lib/python3.11/site-packages/mpyc/runtime.py:127(_exchange_shares) +/lib/python3.11/site-packages/lib/weblooperBench.py:55(call_soon) <- 728195 18.071 41.659 /lib/python3.11/site-packages/mpyc/asyncoro.py:396(typed_asyncoro) + 2 0.000 0.000 /lib/python311.zip/pyodide/webloop.py:428(create_task) + 728197 3.992 13.588 {method 'run' of '_contextvars.Context' objects} + 1261774 5.638 16.184 {method 'set_result' of '_asyncio.Future' objects} +./py/benches/profile.py:5(main) <- 230 0.005 70.755 /lib/python3.11/site-packages/lib/rstats/rstats/prof.py:19(_aprof) +/lib/python3.11/site-packages/cnnmnist.py:129(main) <- 231 0.550 70.837 ./py/benches/profile.py:5(main) +/lib/python3.11/site-packages/lib/api/run.py:68(run_code) <- 230 0.001 70.758 {method 'run' of '_contextvars.Context' objects} +/lib/python311.zip/_pyodide/_base.py:505(eval_code_async) <- 230 0.000 70.758 /lib/python3.11/site-packages/lib/api/run.py:68(run_code) +/lib/python311.zip/_pyodide/_base.py:350(run_async) <- 230 0.000 70.757 /lib/python311.zip/_pyodide/_base.py:505(eval_code_async) +./py/benches/profile.py:1() <- 230 0.001 70.757 /lib/python311.zip/_pyodide/_base.py:350(run_async) +/lib/python3.11/site-packages/lib/rstats/rstats/prof.py:19(_aprof) <- 230 0.001 70.756 ./py/benches/profile.py:1() + + diff --git a/benches/secretsanta.txt b/benches/secretsanta.txt index af4900e2..ce3b756f 100755 --- a/benches/secretsanta.txt +++ b/benches/secretsanta.txt @@ -1,5 +1,16 @@ Python 3.11 +╭────────────────────────────── stats ──────────────────────────────╮ +│ asyncio: tasks: 2 / 1 / 148 / 18 k | loop: 29 k / 92 k / 92 k │ +│ data s/r: 203.7 KiB / 203.7 KiB │ +│ latency: 1.00 ms / 1.64 ms / 34.0 ms │ +│ messages s/r: 11 k / 11 k │ +│ on_runtime_message │ +│ └── time: 4.00 μs / 40.6 μs / 1.78 ms │ +│ send_runtime_message │ +│ └── time: 19.0 μs / 45.3 μs / 8.40 ms │ +╰──────────────────────────── 00:00:07 ─────────────────────────────╯ + ############################# secretsanta/3 ########################## @@ -7,20 +18,23 @@ mswsl1/native 00:01.5 mswin1/native 00:01.8 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 00:10 +mswin1/web/v0.9.2-312c118(wl10)/edge/stats 00:07 +mswin1/web/v0.9.2-312c118(wl10)/edge 00:07 + +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 00:10 mswin1/web/v0.9.2-312c118(wl3)/chrm 00:07 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 00:09.7 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 00:09.7 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 00:09.1 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 01:47 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 01:47 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 01:41 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug 00:05 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats 00:05 mswin1/web/v0.9.2-03f1a8a(owl)/chrm 00:05 -mswin1/web/v0.6.0-3b0f9d4/chrm/debug 00:14 -mswin1/web/v0.2.1-a51a93f/chrm/debug 00:22 +mswin1/web/v0.6.0-3b0f9d4/chrm/stats 00:14 +mswin1/web/v0.2.1-a51a93f/chrm/stats 00:22 ############################# secretsanta/1 ############################# @@ -28,39 +42,39 @@ mswsl1/native 00:00.7 mswin1/native 00:00.8 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 00:02.4 +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 00:02.4 mswin1/web/v0.9.2-312c118(wl3)/chrm 00:02.7 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 00:02.5 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 00:02.5 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 00:02.5 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 01:19 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 01:19 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 01:12 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug 00:01.7 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats 00:01.7 mswin1/web/v0.9.2-03f1a8a(owl)/chrm 00:01.7 -mswin1/web/v0.6.0-3b0f9d4/chrm/debug 00:04 -mswin1/web/v0.2.1-a51a93f/chrm/debug 00:04 +mswin1/web/v0.6.0-3b0f9d4/chrm/stats 00:04 +mswin1/web/v0.2.1-a51a93f/chrm/stats 00:04 -############################# secretsanta/1/no async ############################# +############################# secretsanta/0 ############################# mswsl1/native 00:00.5 mswin1/native 00:00.5 -mswin1/web/v0.9.2-312c118(wl3)/chrm/debug 00:01 +mswin1/web/v0.9.2-312c118(wl3)/chrm/stats 00:01 mswin1/web/v0.9.2-312c118(wl3)/chrm 00:01 -mswin1/web/v0.9.2-e110e8c(owl3)/chrm/debug 00:01 +mswin1/web/v0.9.2-e110e8c(owl3)/chrm/stats 00:01 mswin1/web/v0.9.2-e110e8c(owl3)/chrm 00:01 -mswin1/web/v0.9.2-e110e8c(owl2)/chrm/debug 00:01 +mswin1/web/v0.9.2-e110e8c(owl2)/chrm/stats 00:01 mswin1/web/v0.9.2-e110e8c(owl2)/chrm 00:01 -mswin1/web/v0.9.2-03f1a8a(owl)/chrm/debug 00:01 +mswin1/web/v0.9.2-03f1a8a(owl)/chrm/stats 00:01 mswin1/web/v0.9.2-03f1a8a(owl)/chrm 00:00.9 -mswin1/web/v0.6.0-3b0f9d4/chrm/debug 00:01 -mswin1/web/v0.2.1-a51a93f/chrm/debug 00:01 +mswin1/web/v0.6.0-3b0f9d4/chrm/stats 00:01 +mswin1/web/v0.2.1-a51a93f/chrm/stats 00:01 diff --git a/flake.lock b/flake.lock index fa723376..fdddaf04 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1698752305, - "narHash": "sha256-9DSEAjg8Y00LkGMp0rYI9iK+Ci+BtaxCo3Y/QFxNRow=", + "lastModified": 1705774496, + "narHash": "sha256-LZ2eYC9KWYe3HFGW1ih3bcVJo5BMpdTPxTSKT1F1yqg=", "owner": "cachix", "repo": "devenv", - "rev": "dbcacb95a1d43554d236b1d85daffa0febad8110", + "rev": "6048f1e059867a1bd32d8257ba0d9b5ab6c22879", "type": "github" }, "original": { @@ -112,11 +112,11 @@ "systems": "systems_3" }, "locked": { - "lastModified": 1701680307, - "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", "owner": "numtide", "repo": "flake-utils", - "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", "type": "github" }, "original": { @@ -272,11 +272,11 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1702034373, - "narHash": "sha256-Apubv9die/XRBPI0eRFJyvuyGz/wD4YQUQJHRYCRenc=", + "lastModified": 1705086858, + "narHash": "sha256-/QamQ9bgh/hRB8FWZTH/XWG1KMZCzyi9zBhPO30yRx0=", "owner": "cachix", "repo": "nixpkgs-python", - "rev": "1cae686aa92dbccafe74fd242f984c3ec27c0b20", + "rev": "d530242aee46505e2a950b93760bbe7c9f9b2442", "type": "github" }, "original": { @@ -319,11 +319,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1702151865, - "narHash": "sha256-9VAt19t6yQa7pHZLDbil/QctAgVsA66DLnzdRGqDisg=", + "lastModified": 1705856552, + "narHash": "sha256-JXfnuEf5Yd6bhMs/uvM67/joxYKoysyE3M2k6T3eWbg=", "owner": "nixos", "repo": "nixpkgs", - "rev": "666fc80e7b2afb570462423cb0e1cf1a3a34fedd", + "rev": "612f97239e2cc474c13c9dafa0df378058c5ad8d", "type": "github" }, "original": { @@ -335,16 +335,16 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1685974512, - "narHash": "sha256-WLPHpe96RbPRO9iDtCxgsYkadTheRq7wqXWdGpR6g7w=", - "owner": "domenkozar", + "lastModified": 1704874635, + "narHash": "sha256-YWuCrtsty5vVZvu+7BchAxmcYzTMfolSPP5io8+WYCg=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "1102477695918daba466123cc2ef694ed3a49939", + "rev": "3dc440faeee9e889fe2d1b4d25ad0f430d449356", "type": "github" }, "original": { - "owner": "domenkozar", - "ref": "cpython-moduralize", + "owner": "NixOS", + "ref": "nixos-23.11", "repo": "nixpkgs", "type": "github" } @@ -385,11 +385,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1702365004, - "narHash": "sha256-IRFvmyP1uk1hchRVxaXTqu6YoZCvMM/NVtUf2hD2Tag=", + "lastModified": 1705060653, + "narHash": "sha256-puYyylgrBS4AFAHeyVRTjTUVD8DZdecJfymWJe7H438=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "c12ac880114d52a3cad5fa02b00f2e2090e89982", + "rev": "e0b44e9e2d3aa855d1dd77b06f067cd0e0c3860d", "type": "github" }, "original": { diff --git a/justfile b/justfile index 14fdc9e8..efe44607 100644 --- a/justfile +++ b/justfile @@ -116,8 +116,11 @@ build: devs2: WSLENV=$WSLENV:VITE_HTTPS_KEY:VITE_HTTPS_CERT op.exe run --env-file="./.env" -- wsl env | grep VITE +# WSLENV=$WSLENV:VITE_HTTPS_KEY:VITE_HTTPS_CERT op.exe run --no-masking --env-file="./.env" -- wsl.exe -- echo ${PATH@Q} devs: - WSLENV=$WSLENV:VITE_HTTPS_KEY:VITE_HTTPS_CERT:PATH/p op.exe run --no-masking --env-file="./.env" -- wsl.exe -- VITE_HTTPS=true yarn dev + # WSLENV=$WSLENV:VITE_HTTPS_KEY:VITE_HTTPS_CERT op.exe run --no-masking --env-file="./.env" -- wsl.exe -- export PATH="$PATH" && VITE_HTTPS=true yarn dev + # WSLENV=$WSLENV:VITE_HTTPS_KEY:VITE_HTTPS_CERT:PATH/p op.exe run --no-masking --env-file="./.env" -- wsl.exe -- VITE_HTTPS=true yarn dev + VITE_HTTPS=true op-run yarn dev watch: # wsl.exe -- nix develop --impure -c -- yarn vite --debug diff --git a/mpyc-web-core/lib/transports/PeerJS.ts b/mpyc-web-core/lib/transports/PeerJS.ts index d05c1f15..3bf1d9dd 100644 --- a/mpyc-web-core/lib/transports/PeerJS.ts +++ b/mpyc-web-core/lib/transports/PeerJS.ts @@ -19,20 +19,20 @@ export class PeerJSTransport extends Emittery implements Transp secure: true, // host: "mpyc-demo--headscale-ams3-c99f82e5.demo.mpyc.tech", // port: 443, - config: { - iceServers: [ - { urls: "stun:stun.l.google.com:19302" }, - { - urls: [ - "turn:eu-0.turn.peerjs.com:3478", - // "turn:us-0.turn.peerjs.com:3478", - ], - username: "peerjs", - credential: "peerjsp", - }, - ], - sdpSemantics: "unified-plan", - } + // config: { + // iceServers: [ + // { urls: "stun:stun.l.google.com:19302" }, + // { + // urls: [ + // "turn:eu-0.turn.peerjs.com:3478", + // // "turn:us-0.turn.peerjs.com:3478", + // ], + // username: "peerjs", + // credential: "peerjsp", + // }, + // ], + // sdpSemantics: "unified-plan", + // } // pingInterval: 2345, }; diff --git a/mpyc-web-demo/index.html b/mpyc-web-demo/index.html index 8d207824..f3d6eec8 100644 --- a/mpyc-web-demo/index.html +++ b/mpyc-web-demo/index.html @@ -140,6 +140,7 @@
MPC Demos:
+ diff --git a/mpyc-web-py/benches/profile.py b/mpyc-web-py/benches/profile.py new file mode 100644 index 00000000..7887ff2b --- /dev/null +++ b/mpyc-web-py/benches/profile.py @@ -0,0 +1,13 @@ +import cnnmnist as demo +from mpyc.runtime import mpc + + +async def main(): + """ + Runs the main function of the program, which executes the secretsanta module. + """ + await demo.main() + + +global profile +profile = await prof(main) diff --git a/mpyc-web-py/benches/stats.py b/mpyc-web-py/benches/stats.py index bd067446..0acc9457 100644 --- a/mpyc-web-py/benches/stats.py +++ b/mpyc-web-py/benches/stats.py @@ -20,7 +20,8 @@ def stats_add(path: str, value=1, prefix="asyncio."): @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_time(path: str, value=1, prefix="asyncio."): pass @@ -32,13 +33,15 @@ def stats_acc(path: str, value=1, prefix="asyncio."): @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_time_maybe_send(path: str, value=1, prefix="asyncio."): async_proxy.maybe_send_stats() @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_time_send(path: str, value=1, prefix="asyncio."): async_proxy.send_stats() @@ -50,21 +53,24 @@ def loop(path: str, value=1, prefix="asyncio."): @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_loop(path: str, value=1, prefix="asyncio."): for i in range(1000): pass @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_time_maybe_send_loop(path: str, value=1, prefix="asyncio."): for i in range(1000): async_proxy.maybe_send_stats() @bench -@stats.acc(lambda path, value=1, prefix="asyncio.": stats.time() | {"test2": 1}) +@stats.time() +@stats.acc(lambda path, value=1, prefix="asyncio.": {"test2": 1}) def stats_acc_time_send_loop(path: str, value=1, prefix="asyncio."): for i in range(1000): async_proxy.send_stats() diff --git a/mpyc-web-py/lib/api/proxy.py b/mpyc-web-py/lib/api/proxy.py index 78334c7a..f17250ab 100644 --- a/mpyc-web-py/lib/api/proxy.py +++ b/mpyc-web-py/lib/api/proxy.py @@ -74,7 +74,6 @@ def load_env(self): class AsyncRuntimeProxy: - _postMessage: Callable[[Any], Awaitable[None]] on_ready_message: Callable[[int, str], None] on_runtime_message: Callable[[int, JsProxy], None] on_run_mpc: Callable[[Any], None] @@ -83,8 +82,8 @@ class AsyncRuntimeProxy: def __init__(self, chan: Any): self.chan = chan + self.postMessage = chan.postMessage chan.onmessage = self.onmessage - self._postMessage = chan.postMessage async def fetch(self, filename: str, **opts): return pyfetch(filename, **opts) @@ -99,7 +98,7 @@ def notify_runtime_ready(self): self.postMessage(to_js(["proxy:js:runtime:ready"])) def _send_stats(self): - self._postMessage(to_js(["proxy:js:display:stats", str(stats.to_tree())])) + self.postMessage(to_js(["proxy:js:display:stats", str(stats.to_tree())])) self.latest_stats_update = time.time() def send_stats(self): @@ -111,17 +110,17 @@ def maybe_send_stats(self, duration=1): self._send_stats() return {} - def postMessage(self, *args, **kwargs): - self._postMessage(*args, **kwargs) - # self.maybe_send_stats() + # def postMessage(self, *args, **kwargs): + # self._postMessage(*args, **kwargs) + # # self.maybe_send_stats() - def onmessage(self, event: ProxyEvent): - try: - self._onmessage(event) - except Exception as e: - logger.error(e, exc_info=True, stack_info=True) + # def onmessage(self, event: ProxyEvent): + # try: + # self._onmessage(event) + # except Exception as e: + # logger.error(e, exc_info=True, stack_info=True) - def _onmessage(self, event: ProxyEvent): + def onmessage(self, event: ProxyEvent): # js.console.error("onmessage") # [message_type, *rest] = event.data.to_py() @@ -155,7 +154,8 @@ def _onmessage(self, event: ProxyEvent): case ProxyEventType.MPC_RUNTIME: pid, message = rest - message, ts = message.to_py() + message, ts = message + message = message.to_py() self.on_runtime_message(pid, message, ts) # loop.create_task(self.on_runtime_message(pid, message)) diff --git a/mpyc-web-py/lib/api/run.py b/mpyc-web-py/lib/api/run.py index a7623022..043df726 100644 --- a/mpyc-web-py/lib/api/run.py +++ b/mpyc-web-py/lib/api/run.py @@ -13,6 +13,9 @@ logger = logging.getLogger(__name__) from lib.log import * # # add imports here to make them available by default to the demos run inside the pyodide runtime +from lib.rstats.rstats.prof import prof + +__all__ = ["run_code", "prof"] async def run_file(file: str): diff --git a/mpyc-web-py/lib/rstats/rstats/__init__.py b/mpyc-web-py/lib/rstats/rstats/__init__.py index 0c30550e..05b6cf14 100644 --- a/mpyc-web-py/lib/rstats/rstats/__init__.py +++ b/mpyc-web-py/lib/rstats/rstats/__init__.py @@ -1,7 +1,10 @@ from .bench import bench -from .stats import BaseStatsCollector, DeepCounter, MovingAverage, NestedDict, format_count, format_file_size, format_time +from .prof import prof +from .stats import BaseStatsCollector, DeepCounter, MovingAverage, NestedDict, TimeStats, format_count, format_file_size, format_time __all__ = [ + "prof", + "TimeStats", "BaseStatsCollector", "NestedDict", "bench", diff --git a/mpyc-web-py/lib/rstats/rstats/prof.py b/mpyc-web-py/lib/rstats/rstats/prof.py new file mode 100644 index 00000000..bf8c79a9 --- /dev/null +++ b/mpyc-web-py/lib/rstats/rstats/prof.py @@ -0,0 +1,36 @@ +import asyncio +from cProfile import Profile +from pstats import SortKey, Stats + +__all__ = ["prof"] + + +def process_stats(stats: Stats, amount=30): + stats.sort_stats(SortKey.TIME) + stats.print_stats(amount) + stats.print_callers(amount) + stats.dump_stats("/stats/dump.txt") + stats.sort_stats(SortKey.CUMULATIVE) + stats.print_stats(amount) + stats.print_callers(amount) + stats.dump_stats("/stats/dump2.txt") + + +async def _aprof(func, amount=30): + with Profile() as profile: + await func() + process_stats(Stats(profile), amount) + return profile + + +def _prof(func, amount=20): + with Profile() as profile: + func() + process_stats(Stats(profile), amount) + return profile + + +def prof(func, amount=20): + if asyncio.iscoroutinefunction(func): + return _aprof(func, amount) + return _prof(func, amount) diff --git a/mpyc-web-py/lib/rstats/rstats/stats.py b/mpyc-web-py/lib/rstats/rstats/stats.py index 9fc3bd1e..ee71744f 100644 --- a/mpyc-web-py/lib/rstats/rstats/stats.py +++ b/mpyc-web-py/lib/rstats/rstats/stats.py @@ -16,13 +16,16 @@ """ import asyncio +import inspect import io import json import logging import math +import sys import time from ast import Num from collections import deque +from contextlib import ContextDecorator from datetime import datetime from functools import wraps from typing import Any, Callable, Generic, ParamSpec, TypeVar @@ -54,6 +57,8 @@ # pyright: ignore=all # type: ignore=all +from enum import StrEnum, auto + class MovingAverage: def __init__(self, maxlen: int = 100): @@ -77,6 +82,31 @@ def __str__(self): return str(float(self)) +class TimeStats: + def __init__(self): + self.min = None + self.max = 0 + self.avg = MovingAverage(maxlen=200) + self.total: float = 0 + + def update(self, t: float): + self.total += t + + self.avg.append(t) + + if self.min is None or self.min > t: + self.min = t + + if self.max is None or self.max < t: + self.max = t + + def __str__(self): + return f"∧ {_format_time(self.min)} / μ {_format_time(float(self.avg))} / ∨ {_format_time(self.max)} / ∑ {_format_time(self.total)}" + + def __repr__(self): + return str(self) + + class DeepCounter(NestedDict[K, Numeric | Any]): """A nested dictionary that stores numeric values and supports recursive updates. @@ -175,7 +205,7 @@ def metric(value: float, unit: str = "", precision: int = 3) -> str: ordinal_ = "mμnpfazyrq"[(-exponent - 1) // 3] else: ordinal_ = "" - value_ = format(value, ".%if" % max(0, precision - (exponent % 3) - 1)) + value_ = format(value, ".%if" % max(0, precision - (exponent % 3) - 1)).rstrip("0").rstrip(".") if not (unit or ordinal_) or unit in ("°", "′", "″"): space = "" else: @@ -204,12 +234,21 @@ def time_delta_fmt(time_a, time_b): return datetime.utcfromtimestamp((time_a - time_b).total_seconds()).strftime("%X") -def format_time(time): - # return f'{_format_time(time["min"])} / {_format_time(time["mavg"])} / {_format_time(time["avg"])} / {_format_time(time["max"])}' - return f'{_format_time(time["min"])} / {_format_time(float(time["avg"]))} / {_format_time(time["max"])}' +def format_time(time: TimeStats): + return str(time) + + +def format_times(times): + txt = "" + for k, time in sorted(times.items()): + txt += f"\n\t{k}: {format_time(time)}" + return txt -def _format_time(time: Numeric): +def _format_time(time: Numeric | None): + if time is None: + return "-" + return metric(time / 1_000_000, "s") @@ -218,7 +257,7 @@ def format_file_size(size): def format_count(count, unit=""): - return metric(count, unit=unit, precision=0) + return metric(count, unit=unit, precision=3) # return humanize.intword(count) @@ -241,7 +280,7 @@ def __init__(self, formatters: dict[str, Callable[[str], str]] = {}): self.stats = DeepCounter[str]({}) self.loop = asyncio.get_event_loop() self.formatters = { - "time": format_time, + TIME_ARG: format_times, } | formatters def set_path(self, path: str, value: Any): @@ -253,7 +292,7 @@ def acc_path(self, path: str, value: Any): self.stats.acc_path(path, value) def dec( - self, counter_func: Callable[P, NestedDict[str, Numeric]], update_func: Callable[[], Callable[[NestedDict[str, Numeric]], None]] + self, make_stats_func: Callable[P, NestedDict[str, Numeric]], update_func: Callable[[], Callable[[NestedDict[str, Numeric]], None]] ) -> Callable[[Callable[P, R]], Callable[P, R]]: """ A decorator function for collecting statistics. @@ -271,39 +310,17 @@ def decorator(func: Callable[P, R]) -> Callable[P, R]: def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: if not self.enabled: return func(*args, **kwargs) - - start_time = time.perf_counter_ns() + new_stats: NestedDict[str, Numeric] = make_stats_func(*args, **kwargs) res = func(*args, **kwargs) - elapsed_time = (time.perf_counter_ns() - start_time) // 1000 - - d: NestedDict[str, Numeric] = counter_func(*args, **kwargs) - if FUNC_ARG in d: + if FUNC_ARG in new_stats: f_name = func.__name__ - func_stats = d.pop(FUNC_ARG) - d |= {f_name: func_stats} + func_stats = new_stats.pop(FUNC_ARG) + new_stats |= {f_name: func_stats} if f_name not in self.stats: # pyright: ignore self.stats[f_name] = {} # pyright: ignore - if "time" not in self.stats[f_name]: # pyright: ignore - self.stats[f_name]["time"] = { # pyright: ignore - "min": None, - "max": None, - "avg": MovingAverage(maxlen=200), - } - - self.stats[f_name]["time"]["avg"].append(elapsed_time) # pyright: ignore - self.stats[f_name]["time"]["min"] = ( # pyright: ignore - min(elapsed_time, self.stats[f_name]["time"]["min"]) # pyright: ignore - if self.stats[f_name]["time"]["min"] # pyright: ignore - else elapsed_time # pyright: ignore - ) - self.stats[f_name]["time"]["max"] = ( # pyright: ignore - max(elapsed_time, self.stats[f_name]["time"]["max"]) # pyright: ignore - if self.stats[f_name]["time"]["max"] # pyright: ignore - else elapsed_time # pyright: ignore - ) - update_func()(d) + update_func()(new_stats) return res return wrapper @@ -334,6 +351,98 @@ def acc(self, counter_func: Callable[P, NestedDict[str, Numeric]]) -> Callable[[ """ return self.dec(counter_func, lambda: self.stats.update) + def time(self, label=None, test=1): + return self._timer(self, label, test) + + class _timer(ContextDecorator): + def __init__(self, stats: "BaseStatsCollector", label=None, test=1): + self.stats = stats + self.label = label + + def __call__(self, func, *args, **kwargs): # only called when used as a decorator + self.func = func + if self.label is None: # Label was not provided + self.label = func.__name__ + return super().__call__(func) + + def __enter__(self): + if not self.stats.enabled: + return + + if self.label is None: # Label was not provided + f = sys._getframe().f_back + f_name = f.f_code.co_name + line = f.f_lineno + self.label = f"{f_name}:{line}" or "default" + + if TIME_ARG not in self.stats.stats: # pyright: ignore + self.stats.stats[TIME_ARG] = {} # pyright: ignore + + if self.label not in self.stats.stats[TIME_ARG]: # pyright: ignore + self.stats.stats[TIME_ARG][self.label] = TimeStats() # pyright: ignore + + self.start_time = time.perf_counter_ns() + + def __exit__(self, *args): + if not self.stats.enabled: + return + + elapsed_time = (time.perf_counter_ns() - self.start_time) // 1000 # pyright: ignore + + time_stats: TimeStats = self.stats.stats[TIME_ARG][self.label] # pyright: ignore + time_stats.update(elapsed_time) + + def time2(self) -> Callable[[Callable[P, R]], Callable[P, R]]: + """ + A decorator function for collecting statistics. + + Args: + counter_func (Callable[P, NestedDict[str, Numeric]]): A function that returns a dictionary of statistics. + ff (Callable[[], Callable[[NestedDict[str, Numeric]], None]]): A function that returns a function for updating the statistics. + + Returns: + Callable[[Callable[P, R]], Callable[P, R]]: A decorated function that collects statistics. + """ + + def decorator(func: Callable[P, R]) -> Callable[P, R]: + f_name = func.__name__ + + @wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + if not self.enabled: + return func(*args, **kwargs) + + if TIME_ARG not in self.stats: # pyright: ignore + self.stats[TIME_ARG] = {} # pyright: ignore + + if f_name not in self.stats[TIME_ARG]: # pyright: ignore + self.stats[TIME_ARG][f_name] = { # pyright: ignore + "min": None, + "max": None, + "avg": MovingAverage(maxlen=200), + } + + time_stats = self.stats[TIME_ARG][f_name] # pyright: ignore + + start_time = time.perf_counter_ns() + res = func(*args, **kwargs) + elapsed_time = (time.perf_counter_ns() - start_time) // 1000 # pyright: ignore + + time_stats["avg"].append(elapsed_time) # pyright: ignore + + if time_stats["min"] is None or time_stats["min"] > elapsed_time: # pyright: ignore + time_stats["min"] = elapsed_time # pyright: ignore + + if time_stats["max"] is None or time_stats["max"] < elapsed_time: # pyright: ignore + time_stats["max"] = elapsed_time # pyright: ignore + # self.stats[TIME_ARG][f_name] = time_stats # pyright: ignore + + return res + + return wrapper + + return decorator + def reset(self): """ Resets the statistics counter and enables statistics collection. @@ -388,18 +497,6 @@ def dump(self, name, stats_data, level=logging.DEBUG): """ logger.log(level, self.dumps(name, stats_data), stacklevel=2) - def time(self) -> NestedDict[str, float]: - """ - A decorator function for accumulating statistics. - - Args: - counter_func (Callable[P, NestedDict[str, Numeric]]): A function that returns a dictionary of statistics. - - Returns: - Callable[[Callable[P, R]], Callable[P, R]]: A decorated function that accumulates statistics. - """ - return {FUNC_ARG: {}} - # def avg(self, name: str, value: Numeric = 1): # """ # A decorator function for accumulating statistics. diff --git a/mpyc-web-py/lib/stats.py b/mpyc-web-py/lib/stats.py index 0140db14..290f0270 100644 --- a/mpyc-web-py/lib/stats.py +++ b/mpyc-web-py/lib/stats.py @@ -39,7 +39,7 @@ from rich.text import Text from rich.tree import Tree -from .rstats.rstats import BaseStatsCollector, MovingAverage, format_count, format_file_size, format_time +from .rstats.rstats import BaseStatsCollector, MovingAverage, TimeStats, format_count, format_file_size, format_time # type: ignore-all logger = logging.getLogger(__name__) @@ -65,9 +65,9 @@ def format_asyncio_stats(stats): return ( - f"tasks: {format_count(stats['ntodo'])} / {format_count(stats['tasks'])} / {format_count(stats['max_tasks'])} /" - f" {format_count(stats['total_tasks_count'])} | loop: {format_count(stats['loop_iters'])} /" - f" {format_count(stats['loop_inner_iters'])} / {format_count(stats['call_soon_count'])}" + f"tasks: {format_count(stats['tasks'])} / ∨ {format_count(stats['max_tasks'])} / ∑" + f" {format_count(stats['total_tasks_count'])} | loop: o {format_count(stats['loop_iters'])} / i" + f" {format_count(stats['loop_inner_iters'])} / {format_count(stats['ntodo'])}" ) @@ -79,7 +79,7 @@ def format_data(data): if "received" in data: received = data["received"] - return f"{format_file_size(sent)} / {format_file_size(received)}" + return f"⬆{format_file_size(sent)} / ⬇{format_file_size(received)}" def format_messages(messages): @@ -90,7 +90,7 @@ def format_messages(messages): if "received" in messages: received = messages["received"] - return f"{format_count(sent)} / {format_count(received)}" + return f"⬆{format_count(sent)} / ⬇{format_count(received)}" def metric(value: float, unit: str = "", precision: int = 3) -> str: @@ -137,9 +137,9 @@ def __init__(self): formatters={ "total_bytes_received": format_file_size, "total_bytes_sent": format_file_size, - "messages s/r": format_messages, + "messages": format_messages, "asyncio": format_asyncio_stats, - "data s/r": format_data, + "data": format_data, "latency": format_time, } ) @@ -152,15 +152,15 @@ def to_tree(self): self.stats["gc"] = self.gc_stats() return super().to_tree() - def asyncio_stats(self): - if "asyncio" not in self.stats: + def asyncio_stats(self, key="asyncio"): + if key not in self.stats: return tasks = len(asyncio.tasks._all_tasks) - if tasks > self.stats["asyncio"]["max_tasks"]: - self.stats["asyncio"]["max_tasks"] = tasks + if tasks > self.stats[key]["max_tasks"]: + self.stats[key]["max_tasks"] = tasks - self.stats["asyncio"]["tasks"] = tasks + self.stats[key]["tasks"] = tasks # return { # "total_tasks_count": getattr(self.loop, "total_tasks_count", 0), @@ -194,15 +194,6 @@ def reset(self): # self.enabled = logging.root.getEffectiveLevel() <= logging.DEBUG def stat(self, s: NestedDict[str, float]) -> NestedDict[str, float]: - """ - Compute statistics on the input dictionary. - - Args: - s (NestedDict[str, float]): A nested dictionary of float values. - - Returns: - NestedDict[str, float]: A nested dictionary of statistics computed on the input dictionary. - """ return s def latency(self, ts: int, key="latency") -> NestedDict[str, float]: # pyright: ignore @@ -211,15 +202,9 @@ def latency(self, ts: int, key="latency") -> NestedDict[str, float]: # pyright: return {} if key not in self.stats: - self.stats[key] = { # pyright: ignore - "min": None, - "max": 0, - "avg": MovingAverage(maxlen=200), - } + self.stats[key] = TimeStats() # pyright: ignore - self.stats[key]["avg"].append(l) - self.stats[key]["min"] = min(l, self.stats[key]["min"]) if self.stats[key]["min"] else l # pyright: ignore - self.stats[key]["max"] = max(l, self.stats[key]["max"]) if self.stats[key]["max"] else l # pyright: ignore + self.stats[key].update(l) return {} @@ -236,10 +221,10 @@ def sent_to(self, pid: int, msg: bytes) -> NestedDict[str, float]: """ return { - "messages s/r": { + "messages": { "sent": +1, }, - "data s/r": { + "data": { "sent": +len(msg), }, # "messages": +1, @@ -264,10 +249,10 @@ def received_from(self, pid: int, msg: bytes) -> NestedDict[str, float]: - bytes_received_from[pid]: The number of bytes received from the given party. """ return { - "messages s/r": { + "messages": { "received": +1, }, - "data s/r": { + "data": { "received": +len(msg), }, # "total_messages_received": +1, diff --git a/mpyc-web-py/lib/weblooper.py b/mpyc-web-py/lib/weblooper.py index 16b6ac85..b0211529 100644 --- a/mpyc-web-py/lib/weblooper.py +++ b/mpyc-web-py/lib/weblooper.py @@ -1,4 +1,5 @@ # import .weblooperV2 -from .weblooperV10 import * +# from .weblooperV10 import * +from .weblooperBench import * # from .webloop_pyodide_old import * diff --git a/mpyc-web-py/lib/weblooperBench.1.py b/mpyc-web-py/lib/weblooperBench.1.py new file mode 100644 index 00000000..865c6948 --- /dev/null +++ b/mpyc-web-py/lib/weblooperBench.1.py @@ -0,0 +1,142 @@ +import asyncio +import collections +import contextvars +import types +from functools import partial, partialmethod +from random import sample, shuffle +from typing import Any, Callable, Optional + +import js +import rich +from lib.api import async_proxy +from lib.stats import MovingAverage, stats +from pyodide.code import run_js +from pyodide.ffi import IN_BROWSER, create_once_callable, create_proxy +from pyodide.webloop import PyodideFuture, PyodideTask, WebLoop + + +class WebLooper(WebLoop): + def __init__(self): + super().__init__() + stats.reset() + old_add = asyncio.tasks._all_tasks.add + + # @stats.time() + def add(self, task): + stats_add("total_tasks_count") + old_add(task) + + # asyncio.tasks._all_tasks.add = types.MethodType(add, asyncio.tasks._all_tasks) + # stats_set("total_tasks_count", len(asyncio.tasks._all_tasks)) + + self.running = False + self._ready = collections.deque() + self._run_once_proxy = create_proxy(self._run_once) + self.chan = js.MessageChannel.new() + self.chan.port1.onmessage = self.run_once_proxy + + # @stats.time("run_handle_outer2") + # @stats.time("run_handle_outer1") + # @stats.time() + def run_handle(self, h): + if h._cancelled: + return + try: + h._run() + except SystemExit as e: + if self._system_exit_handler: + self._system_exit_handler(e.code) + else: + raise + except KeyboardInterrupt: + if self._keyboard_interrupt_handler: + self._keyboard_interrupt_handler() + else: + raise + + # @stats.time() + def run_once_proxy(self, *args, **kwargs): + self._run_once_proxy(*args, **kwargs) + + @stats.time("nothing3") + @stats.time("nothing2") + @stats.time() + def nothing(self): + pass + + def call_soon( + self, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ) -> asyncio.Handle: + # self.nothing() + + delay = 0 + return self.call_later(delay, callback, *args, context=context) + + # @stats.time() + def call_later( # type: ignore[override] + self, + delay: float, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ): + h, run = self.make_handle(callback, args, context=context) + # with stats.time(): + + if delay < 0: + raise ValueError("Can't schedule in the past") + + if delay == 0: + self._ready_append(run) + + if not self.running: + self.running = True + self.trigger_run_once() + + return h + call_handle_later(run, delay) + return h + + def _ready_append(self, handle): + self._ready.append(handle) + + def trigger_run_once(self): + self.chan.port2.postMessage(None) + + # @stats.time() + def _run_once(self, *args, **kwargs): + # stats_add("loop_iters") + ntodo = len(self._ready) + # stats_set("ntodo", ntodo) + + # with stats.time(): + for _ in range(ntodo): + # stats_add("loop_inner_iters") + self._ready.popleft()() + + nleft = len(self._ready) + # ntodoz(nleft, "left") + if nleft == 0: + self.running = False + else: + self.trigger_run_once() + + # @stats.time() + def make_handle(self, callback, args, context=None): + h = asyncio.Handle(callback, args, self, context=context) + return h, partial(self.run_handle, h) + + +def call_handle_later(handle, delay): + js.setTimeout(create_once_callable(handle), delay * 1000) + + +def stats_add(path: str, value=1, prefix="asyncio."): + stats.acc_path(f"{prefix}{path}", value) + + +def stats_set(path: str, value=1, prefix="asyncio."): + stats.set_path(f"{prefix}{path}", value) diff --git a/mpyc-web-py/lib/weblooperBench.py b/mpyc-web-py/lib/weblooperBench.py new file mode 100644 index 00000000..a44e093b --- /dev/null +++ b/mpyc-web-py/lib/weblooperBench.py @@ -0,0 +1,99 @@ +import asyncio +import collections +import contextvars +import types +from functools import partial, partialmethod +from random import sample, shuffle +from typing import Any, Callable, Optional + +import js +import rich +from lib.api import async_proxy +from lib.stats import MovingAverage, stats +from pyodide.code import run_js +from pyodide.ffi import IN_BROWSER, create_once_callable, create_proxy +from pyodide.webloop import PyodideFuture, PyodideTask, WebLoop + + +class WebLooper(WebLoop): + def __init__(self): + super().__init__() + stats.reset() + + self.running = False + self._ready = collections.deque() + # self.set_on_message, self.trigger_run_once = run_js(""" + self.port1, self.trigger_run_once = run_js(""" + let chan = new MessageChannel(); + let trigger_run_once = () => chan.port2.postMessage(undefined); + + // let set_on_message = (cb) => chan.port1.onmessage = (_) => {cb()}; + // [set_on_message, post] + [chan.port1, trigger_run_once] + """) + # self.set_on_message(create_proxy(self._run_once)) + self.port1.onmessage = self._run_once + + def call_soon( + self, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ) -> asyncio.Handle: + delay = 0 + return self.call_later(delay, callback, *args, context=context) + + # @stats.time() + def call_later( # type: ignore[override] + self, + delay: float, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ): + h = asyncio.Handle(callback, args, self, context=context) + # with stats.time(): + + if delay < 0: + raise ValueError("Can't schedule in the past") + + if delay == 0: + self._ready.append(h) + + if not self.running: + self.running = True + self.trigger_run_once() + + return h + js.setTimeout(create_once_callable(lambda: h._run() if not h._cancelled else None), delay * 1000) + return h + + def _ready_append(self, handle): + self._ready.append(handle) + + # @stats.time() + def _run_once(self, *args, **kwargs): + ntodo = len(self._ready) + + for _ in range(ntodo): + h = self._ready.popleft() + if h._cancelled: + continue + h._run() + + if len(self._ready) == 0: + self.running = False + else: + self.trigger_run_once() + + +def call_handle_later(handle, delay): + js.setTimeout(create_once_callable(handle), delay * 1000) + + +def stats_add(path: str, value=1, prefix="asyncio."): + stats.acc_path(f"{prefix}{path}", value) + + +def stats_set(path: str, value=1, prefix="asyncio."): + stats.set_path(f"{prefix}{path}", value) diff --git a/mpyc-web-py/lib/weblooperV10.py b/mpyc-web-py/lib/weblooperV10.py index 0a1d41de..82ebc9fc 100644 --- a/mpyc-web-py/lib/weblooperV10.py +++ b/mpyc-web-py/lib/weblooperV10.py @@ -33,6 +33,7 @@ def add(self, task): self.chan = js.MessageChannel.new() self.chan.port1.onmessage = self._run_once_proxy + # @stats.acc(lambda self, callback, *args, context: stats.time()) def call_soon( self, callback: Callable[..., Any], @@ -70,42 +71,31 @@ def run_handle(): raise if delay == 0: - stats_add("call_soon_count") self._ready.append(run_handle) if not self.running: self.running = True self.trigger_run_once() + # self._run_once() return h js.setTimeout(create_once_callable(run_handle), delay * 1000) return h def trigger_run_once(self): - stats_add("loop_iters") self.chan.port2.postMessage(None) - @stats.acc(lambda *args, **kwargs: stats.time()) + # @stats.acc(lambda *args, **kwargs: stats.time()) def _run_once(self, *args, **kwargs): + stats_add("loop_iters") ntodo = len(self._ready) stats_set("ntodo", ntodo) - ntodoz(ntodo) - async_proxy.maybe_send_stats() - - # sample_indices = set(sample(range(ntodo), ntodo)) - # ready_shuff = [self._ready[i] for i in sample_indices] - # self._ready.clear() - - # for h in ready_shuff: - # stats_add("loop_inner_iters") - # h() - - while self._ready: + for _ in range(ntodo): stats_add("loop_inner_iters") self._ready.popleft()() nleft = len(self._ready) - ntodoz(nleft, "left") + # ntodoz(nleft, "left") if nleft == 0: self.running = False else: @@ -113,6 +103,9 @@ def _run_once(self, *args, **kwargs): def ntodoz(ntodo, key="ready"): + if not stats.enabled: + return + if key not in stats.stats: stats.stats[key] = { # pyright: ignore "min_cnt": 0, diff --git a/mpyc-web-py/lib/weblooperV11.py b/mpyc-web-py/lib/weblooperV11.py new file mode 100644 index 00000000..92f6ff1b --- /dev/null +++ b/mpyc-web-py/lib/weblooperV11.py @@ -0,0 +1,91 @@ +import asyncio +import collections +import contextvars +import types +from random import sample, shuffle +from typing import Any, Callable, Optional + +import js +import rich +from lib.api import async_proxy +from lib.stats import MovingAverage, stats +from pyodide.code import run_js +from pyodide.ffi import IN_BROWSER, create_once_callable, create_proxy +from pyodide.webloop import PyodideFuture, PyodideTask, WebLoop + + +class WebLooper(WebLoop): + def __init__(self): + super().__init__() + stats.reset() + old_add = asyncio.tasks._all_tasks.add + + def add(self, task): + stats_add("total_tasks_count") + old_add(task) + + asyncio.tasks._all_tasks.add = types.MethodType(add, asyncio.tasks._all_tasks) + stats_set("total_tasks_count", len(asyncio.tasks._all_tasks)) + + self.running = False + self._ready = collections.deque() + + @stats.acc(lambda self, callback, *args, context: stats.time()) + def call_soon( + self, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ) -> asyncio.Handle: + delay = 0 + return self.call_later(delay, callback, *args, context=context) + + def call_later( # type: ignore[override] + self, + delay: float, + callback: Callable[..., Any], + *args: Any, + context: contextvars.Context | None = None, + ) -> asyncio.Handle: + if delay < 0: + raise ValueError("Can't schedule in the past") + h = asyncio.Handle(callback, args, self, context=context) + + def run_handle(): + if h._cancelled: + return + try: + h._run() + except SystemExit as e: + if self._system_exit_handler: + self._system_exit_handler(e.code) + else: + raise + except KeyboardInterrupt: + if self._keyboard_interrupt_handler: + self._keyboard_interrupt_handler() + else: + raise + + if delay == 0: + self._ready.append(run_handle) + if not self.running: + self.running = True + stats_add("loop_iters") + while self._ready: + stats_add("loop_inner_iters") + stats_set("ntodo", len(self._ready)) + self._ready.popleft()() + self.running = False + + return h + js.setTimeout(create_once_callable(run_handle), delay * 1000) + return h + + +def stats_add(path: str, value=1, prefix="asyncio."): + stats.acc_path(f"{prefix}{path}", value) + + +def stats_set(path: str, value=1, prefix="asyncio."): + stats.set_path(f"{prefix}{path}", value) diff --git a/mpyc-web-py/mpyc/mpyc/finfields.py b/mpyc-web-py/mpyc/mpyc/finfields.py index 74abde3f..a3f2f655 100644 --- a/mpyc-web-py/mpyc/mpyc/finfields.py +++ b/mpyc-web-py/mpyc/mpyc/finfields.py @@ -12,11 +12,12 @@ Much of the NumPy API can be used to manipulate these arrays as well. """ -import os import functools -from mpyc.numpy import np -from mpyc import gmpy as gmpy2 +import os + from mpyc import gfpx +from mpyc import gmpy as gmpy2 +from mpyc.numpy import np def GF(modulus): @@ -34,7 +35,7 @@ def GF(modulus): if p == 2: n, w = 1, 1 else: - n, w = 2, p-1 + n, w = 2, p - 1 field = pGF(p, n, w) if not np: @@ -46,8 +47,8 @@ def GF(modulus): BaseFFArray = BinaryFieldArray else: # issubclass(field, ExtensionFieldElement) BaseFFArray = ExtensionFieldArray - name = f'Array{field.__name__}' - array = type(name, (BaseFFArray,), {'__slots__': ()}) + name = f"Array{field.__name__}" + array = type(name, (BaseFFArray,), {"__slots__": ()}) array.field = field field.array = array globals()[name] = array # NB: exploit (almost?) unique name dynamic array type @@ -60,7 +61,7 @@ class FiniteFieldElement: Invariant: 'value' is reduced w.r.t. modulus. """ - __slots__ = 'value' + __slots__ = "value" modulus: type # set by subclass order = None @@ -80,22 +81,22 @@ def __array_function__(self, func, types, args, kwargs): def __int__(self): """Extract field element as an integer value.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") @classmethod def to_bytes(cls, x): """Return byte string representing the given list/ndarray of integers x.""" - byte_order = 'little' + byte_order = "little" r = cls.byte_length - return r.to_bytes(2, byte_order) + b''.join(v.to_bytes(r, byte_order) for v in x) + return r.to_bytes(2, byte_order) + b"".join(v.to_bytes(r, byte_order) for v in x) @staticmethod def from_bytes(data): """Return the list of integers represented by the given byte string.""" - byte_order = 'little' + byte_order = "little" from_bytes = int.from_bytes # cache r = from_bytes(data[:2], byte_order) - return [from_bytes(data[i:i+r], byte_order) for i in range(2, len(data), r)] + return [from_bytes(data[i : i + r], byte_order) for i in range(2, len(data), r)] def __add__(self, other): """Addition.""" @@ -218,7 +219,7 @@ def __itruediv__(self, other): def __pow__(self, other): """Exponentiation.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def __lshift__(self, other): """Left shift.""" @@ -242,7 +243,7 @@ def __ilshift__(self, other): def __rshift__(self, other): """Right shift.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def __rrshift__(self, other): """Right shift (with reflected arguments).""" @@ -250,12 +251,12 @@ def __rrshift__(self, other): def __irshift__(self, other): """In-place right shift.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") @classmethod def _reciprocal(cls, a): """Multiplicative inverse.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def reciprocal(self): """Multiplicative inverse.""" @@ -265,7 +266,7 @@ def reciprocal(self): @classmethod def _sqrt(cls, a, INV=False): """Modular (inverse) square root.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def sqrt(self, INV=False): """Modular (inverse) square root.""" @@ -275,7 +276,7 @@ def sqrt(self, INV=False): @classmethod def _is_sqr(cls, a): """Test for quadratic residuosity (0 is also square).""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def is_sqr(self): """Test for quadratic residuosity (0 is also square).""" @@ -317,24 +318,24 @@ def find_prime_root(l, blum=True, n=1): w = 1 else: p = 3 - n, w = 2, p-1 + n, w = 2, p - 1 elif n <= 2: p = gmpy2.prev_prime(1 << l) if blum: - while p%4 != 3: + while p % 4 != 3: p = gmpy2.prev_prime(p) p = int(p) - w = p-1 if n == 2 else 1 + w = p - 1 if n == 2 else 1 else: assert blum if not gmpy2.is_prime(n): n = gmpy2.next_prime(n) - p = 1 + 2*n * (3 + 2*((1 << l-3) // n)) + p = 1 + 2 * n * (3 + 2 * ((1 << l - 3) // n)) while not gmpy2.is_prime(p): - p += 4*n + p += 4 * n a = 2 - while (w := gmpy2.powmod(a, (p-1) // n, p)) == 1: + while (w := gmpy2.powmod(a, (p - 1) // n, p)) == 1: a += 1 p, n, w = int(p), int(n), int(w) return p, n, w @@ -344,10 +345,10 @@ def find_prime_root(l, blum=True, n=1): def pGF(p, n, w): """Create a finite field for given prime modulus p.""" if not gmpy2.is_prime(p): - raise ValueError('modulus is not a prime') + raise ValueError("modulus is not a prime") - GFp = type(f'GF({p})', (PrimeFieldElement,), {'__slots__': ()}) - GFp.__doc__ = 'Class of prime field elements.' + GFp = type(f"GF({p})", (PrimeFieldElement,), {"__slots__": ()}) + GFp.__doc__ = "Class of prime field elements." GFp.modulus = p GFp.order = p GFp.characteristic = p @@ -371,8 +372,8 @@ class PrimeFieldElement(FiniteFieldElement): _mix_types = int def __init__(self, value): - if not isinstance(value, int): - raise TypeError(f'int required, got {type(value).__name__}') + # if not isinstance(value, int): + # raise TypeError(f'int required, got {type(value).__name__}') # Directly call int.__mod__() for efficiency: value = value.__mod__(self.modulus) @@ -385,8 +386,7 @@ def createGF(p, n, w): return PrimeFieldElement.__new__(obj) def __reduce__(self): - return (PrimeFieldElement.createGF, (self.modulus, self.nth, self.root), - (None, {'value': self.value})) + return (PrimeFieldElement.createGF, (self.modulus, self.nth, self.root), (None, {"value": self.value})) def __int__(self): """Extract field element as a (signed) integer value.""" @@ -438,32 +438,32 @@ def _sqrt(cls, a, INV=False): p = cls.modulus if a == 0: if INV: - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") return a if p == 2: return a - if p&3 == 3: + if p & 3 == 3: if INV: - p4 = (p*3 - 5) >> 2 # a**p4 == a**(-1/2) == 1/sqrt(a) mod p + p4 = (p * 3 - 5) >> 2 # a**p4 == a**(-1/2) == 1/sqrt(a) mod p else: - p4 = (p+1) >> 2 + p4 = (p + 1) >> 2 return int(gmpy2.powmod(a, p4, p)) # 1 (mod 4) primes are covered using Cipolla-Lehmer's algorithm. # find b s.t. b^2 - 4*a is not a square b = 1 - while gmpy2.legendre(b * b - 4*a, p) != -1: + while gmpy2.legendre(b * b - 4 * a, p) != -1: b += 1 # compute u*X + v = X^{(p+1)/2} mod f, for f = X^2 - b*X + a u, v = 0, 1 - e = (p+1) >> 1 + e = (p + 1) >> 1 for i in range(e.bit_length() - 1, -1, -1): u2 = (u * u) % p - u = ((u<<1) * v + b * u2) % p + u = ((u << 1) * v + b * u2) % p v = (v * v - a * u2) % p if (e >> i) & 1: u, v = (v + b * u) % p, (-a * u) % p @@ -492,7 +492,7 @@ def unsigned_(self): return self.value def __repr__(self): - return f'{self.__int__()}' + return f"{self.__int__()}" def find_irreducible(p, d): @@ -507,11 +507,11 @@ def xGF(modulus): p = modulus.p poly = gfpx.GFpX(p) if not poly.is_irreducible(modulus): - raise ValueError('modulus is not irreducible') + raise ValueError("modulus is not irreducible") d = poly.deg(modulus) BaseFieldElement = BinaryFieldElement if p == 2 else ExtensionFieldElement - GFq = type(f'GF({p}^{d})', (BaseFieldElement,), {'__slots__': ()}) + GFq = type(f"GF({p}^{d})", (BaseFieldElement,), {"__slots__": ()}) GFq.__doc__ = f'Class of {"binary" if p == 2 else "extension"} field elements.' GFq.modulus = modulus GFq.order = p**d @@ -543,7 +543,7 @@ def createGF(modulus): return ExtensionFieldElement.__new__(obj) def __reduce__(self): - return ExtensionFieldElement.createGF, (self.modulus,), (None, {'value': self.value}) + return ExtensionFieldElement.createGF, (self.modulus,), (None, {"value": self.value}) def __int__(self): return int(self.value) @@ -579,25 +579,25 @@ def _sqrt(cls, a, INV=False): q = cls.order if a == []: if INV: - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") return a - if q%2 == 0: + if q % 2 == 0: q2 = q >> 1 if INV: q2 -= 1 return poly.powmod(a, q2, cls.modulus) - if q&3 == 3: + if q & 3 == 3: if INV: - q4 = (q*3 - 5) >> 2 # a**q4 == a**(-1/2) == 1/sqrt(a) in GF(q) + q4 = (q * 3 - 5) >> 2 # a**q4 == a**(-1/2) == 1/sqrt(a) in GF(q) else: - q4 = (q+1) >> 2 + q4 = (q + 1) >> 2 return poly.powmod(a, q4, cls.modulus) # Tonelli-Shanks - n = q-1 + n = q - 1 s = (n & -n).bit_length() - 1 # number of times 2 divides n t = n >> s # q - 1 = t 2^s, t odd @@ -606,13 +606,13 @@ def _sqrt(cls, a, INV=False): i = 2 while True: z = poly.powmod(i, t, cls.modulus) - if poly.powmod(z, 1 << s-1, cls.modulus) != 1: + if poly.powmod(z, 1 << s - 1, cls.modulus) != 1: break i += 1 cls._least_qnr = z # cache least QNR raised to power t # TODO: improve following code a bit - w = poly.powmod(a, t>>1, cls.modulus) + w = poly.powmod(a, t >> 1, cls.modulus) x = a * w % cls.modulus b = x * w % cls.modulus v = s @@ -636,13 +636,13 @@ def _sqrt(cls, a, INV=False): def _is_sqr(cls, a): poly = type(a) q = cls.order - if q%2 == 0: + if q % 2 == 0: return True - return poly.powmod(a, (q-1) >> 1, cls.modulus) != [poly.p - 1] + return poly.powmod(a, (q - 1) >> 1, cls.modulus) != [poly.p - 1] def __repr__(self): - return f'{self.value}' + return f"{self.value}" class BinaryFieldElement(ExtensionFieldElement): @@ -664,7 +664,7 @@ def _sqrt(cls, a, INV=False): q = cls.order if a == 0: if INV: - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") return a @@ -679,9 +679,10 @@ def _sqrt(cls, a, INV=False): def _implements(numpy_function_name): """Register an __array_function__ implementation.""" + def decorator(func): if np: - numpy_function = eval('np.' + numpy_function_name) + numpy_function = eval("np." + numpy_function_name) _HANDLED_FUNCTIONS[numpy_function] = func return func @@ -704,7 +705,7 @@ class FiniteFieldArray: Invariant: elements of array 'value' are reduced w.r.t. modulus. """ - __slots__ = 'value' + __slots__ = "value" field: type # finite field of the array _mix_types: type # or tuple of types @@ -720,35 +721,34 @@ def __init__(self, value, check=True, copy=False): def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # TODO: make more general cls = type(self) - if any(isinstance(a, np.ndarray) and a.dtype != object and a.dtype not in cls._mix_types - for a in inputs): + if any(isinstance(a, np.ndarray) and a.dtype != object and a.dtype not in cls._mix_types for a in inputs): return NotImplemented - if ufunc.__name__ == 'equal': + if ufunc.__name__ == "equal": return inputs[1].__eq__(inputs[0]) # needed to avoid infinite recursion - if ufunc.__name__ == 'not_equal': + if ufunc.__name__ == "not_equal": return inputs[1].__ne__(inputs[0]) - if ufunc.__name__ == 'left_shift': + if ufunc.__name__ == "left_shift": return inputs[0] << inputs[1] - if ufunc.__name__ == 'right_shift': + if ufunc.__name__ == "right_shift": return inputs[0] >> inputs[1] - if ufunc.__name__ == 'power': + if ufunc.__name__ == "power": if isinstance(inputs[1], cls): return NotImplemented - if ufunc.__name__ == 'reciprocal': + if ufunc.__name__ == "reciprocal": return type(self).reciprocal(inputs[0]) - if ufunc.__name__ == 'sqrt': + if ufunc.__name__ == "sqrt": return type(self).sqrt(inputs[0]) inputs = tuple(a.value if isinstance(a, (cls, cls.field)) else a for a in inputs) a = getattr(ufunc, method)(*inputs, **kwargs) - if method == 'at': # 'at' is in-place so make sure to enforce invariant + if method == "at": # 'at' is in-place so make sure to enforce invariant # TODO: check other methods than '__call__' (at, reduce, reduceat, accumulate, outer) self.value = self.value % cls.field.modulus else: @@ -762,7 +762,7 @@ def __array_function__(self, func, types, args, kwargs): elif isinstance(self, FiniteFieldElement): cls = type(self).array else: - raise TypeError('wrong type for self in __array_function__(self, ...) call') + raise TypeError("wrong type for self in __array_function__(self, ...) call") if func in _HANDLED_FUNCTIONS: return _HANDLED_FUNCTIONS[func](*args, **kwargs) @@ -791,9 +791,9 @@ def __array_function__(self, func, types, args, kwargs): a = func(*args, **kwargs) if isinstance(a, np.ndarray): - if func.__name__ in ('roll', 'diagonal', 'diag_flat'): + if func.__name__ in ("roll", "diagonal", "diag_flat"): a = cls(a, check=False) - elif func.__name__ != 'flatnonzero': + elif func.__name__ != "flatnonzero": a = cls(a) elif isinstance(a, list): # for func like vsplit returning list of arrays @@ -805,22 +805,22 @@ def __array_function__(self, func, types, args, kwargs): return a @property - @_implements('shape') + @_implements("shape") def shape(self): return self.value.shape @property - @_implements('ndim') + @_implements("ndim") def ndim(self): return self.value.ndim @property - @_implements('size') + @_implements("size") def size(self): return self.value.size @staticmethod - @_implements('block') + @_implements("block") def _np_block(arrays): def extract_type(s): if isinstance(s, list): @@ -856,7 +856,7 @@ def __iter__(self): yield a @staticmethod - @_implements('linalg.solve') + @_implements("linalg.solve") def gauss_solve(A, B): """Linear solve by Gaussian elimination on matrix (A | B).""" # TODO: extend to more dimensions, solve in last 2 dimensions @@ -866,41 +866,41 @@ def gauss_solve(A, B): modulus = field.modulus # int or gfpx.Polynomial(0) n = A.shape[0] if not A.shape == (n, n): - raise np.linalg.LinAlgError('array must be square') + raise np.linalg.LinAlgError("array must be square") A = np.concatenate((A.value, B.value), axis=1) # Gaussian elimination for k in range(n): if A[k, k] == 0: - for x in range(k+1, n): + for x in range(k + 1, n): if A[x, k] != 0: break else: - raise ZeroDivisionError('no inverse exists') + raise ZeroDivisionError("no inverse exists") A[[k, x]] = A[[x, k]] A[k, k] = (1 / field(A[k, k])).value # store reciprocal of diagonal elts - A[k+1:, k] = A[k+1:, k] * A[k, k] % modulus - A[k+1:, k+1:] = (A[k+1:, k+1:] - np.outer(A[k+1:, k], A[k, k+1:])) % modulus + A[k + 1 :, k] = A[k + 1 :, k] * A[k, k] % modulus + A[k + 1 :, k + 1 :] = (A[k + 1 :, k + 1 :] - np.outer(A[k + 1 :, k], A[k, k + 1 :])) % modulus # back substitution - A[n-1, n:] = A[n-1, n:] * A[n-1, n-1] % modulus # avoid np.dot on empty dtype='O' arrays - for i in range(n-2, -1, -1): - A[i, n:] = (A[i, n:] - np.dot(A[i, i+1:n], A[i+1:, n:])) * A[i, i] % modulus + A[n - 1, n:] = A[n - 1, n:] * A[n - 1, n - 1] % modulus # avoid np.dot on empty dtype='O' arrays + for i in range(n - 2, -1, -1): + A[i, n:] = (A[i, n:] - np.dot(A[i, i + 1 : n], A[i + 1 :, n:])) * A[i, i] % modulus return cls(A[:, n:], check=False) @staticmethod - @_implements('linalg.inv') + @_implements("linalg.inv") def gauss_inv(A): """Inverse by Gaussian elimination on augmented matrix (A | I).""" # TODO: extend to more dimensions, invert last 2 dimensions - B = type(A)(np.eye(len(A), dtype='O')) + B = type(A)(np.eye(len(A), dtype="O")) return FiniteFieldArray.gauss_solve(A, B) @staticmethod - @_implements('linalg.det') + @_implements("linalg.det") def gauss_det(a): """Determinant by Gaussian elimination on (last 2 dimensions of) array a.""" cls = type(a) @@ -911,14 +911,14 @@ def gauss_det(a): return np.linalg.det(np.empty(a.shape)) n = a.shape[-1] - d = np.empty(a.size // n**2, dtype='O') + d = np.empty(a.size // n**2, dtype="O") for i, A in enumerate(a.value.reshape((-1, n, n))): A = A.copy() # Gaussian elimination for k in range(n): if A[k, k] == 0: - for x in range(k+1, n): + for x in range(k + 1, n): if A[x, k] != 0: break else: @@ -927,8 +927,8 @@ def gauss_det(a): A[[k, x]] = A[[x, k]] inv_k = (1 / field(A[k, k])).value - A[k+1:, k] = A[k+1:, k] * inv_k % modulus - A[k+1:, k+1:] = (A[k+1:, k+1:] - np.outer(A[k+1:, k], A[k, k+1:])) % modulus + A[k + 1 :, k] = A[k + 1 :, k] * inv_k % modulus + A[k + 1 :, k + 1 :] = (A[k + 1 :, k + 1 :] - np.outer(A[k + 1 :, k], A[k, k + 1 :])) % modulus else: d[i] = cls(np.diag(A), check=False).prod().value @@ -941,7 +941,7 @@ def gauss_det(a): return d @staticmethod - @_implements('linalg.matrix_power') + @_implements("linalg.matrix_power") def matrix_pow(A, n): # needed for handling negative n cls = type(A) p = cls.field.modulus @@ -951,7 +951,7 @@ def matrix_pow(A, n): # needed for handling negative n n = -n D = A.value - C = np.eye(len(A), dtype='O') + C = np.eye(len(A), dtype="O") for i in range(n.bit_length() - 1): # D = A^(2^i) holds if (n >> i) & 1: @@ -962,7 +962,7 @@ def matrix_pow(A, n): # needed for handling negative n return cls(C) @staticmethod - @_implements('diag') + @_implements("diag") def diag(a, k=0): cls = type(a) return cls(np.diag(a.value, k), check=False) @@ -1170,7 +1170,7 @@ def __ipow__(self, other): @classmethod def _pow(cls, a, b): """Exponentiation.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def __neg__(self): """Negation.""" @@ -1201,7 +1201,7 @@ def __rtruediv__(self, other): return cls(other * cls._reciprocal(self.value)) def __itruediv__(self, other): - """"In-place division.""" + """ "In-place division.""" other = self._coerce(other) if other is NotImplemented: return NotImplemented @@ -1259,7 +1259,7 @@ def __irshift__(self, other): @classmethod def _reciprocal(cls, a): """Multiplicative inverse.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def reciprocal(self): """Multiplicative inverse.""" @@ -1269,7 +1269,7 @@ def reciprocal(self): @classmethod def _sqrt(cls, a, INV=False): """Modular (inverse) square root.""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def sqrt(self, INV=False): """Modular (inverse) square root.""" @@ -1279,7 +1279,7 @@ def sqrt(self, INV=False): @classmethod def _is_sqr(cls, a): """Test for quadratic residuosity (0 is also square).""" - raise NotImplementedError('abstract method') + raise NotImplementedError("abstract method") def is_sqr(self): """Test for quadratic residuosity (0 is also square).""" @@ -1304,7 +1304,7 @@ def take(self, *args, **kwargs): return type(self)(self.value.take(*args, **kwargs), check=False) def tolist(self): - return np.vectorize(self.field, otypes='O')(self.value).tolist() + return np.vectorize(self.field, otypes="O")(self.value).tolist() def ravel(self, *args, **kwargs): # view -- no copy return type(self)(self.value.ravel(*args, **kwargs), check=False) @@ -1323,15 +1323,15 @@ def sum(self, *args, **kwargs): return type(self)(a, check=True) def prod(self, *args, **kwargs): - v = kwargs.get('initial', 1) + v = kwargs.get("initial", 1) if not isinstance(v, self.field): v = self.field(v) # prevent growth using field multiplication - kwargs['initial'] = v + kwargs["initial"] = v a = self.value.prod(*args, **kwargs) if not isinstance(a, np.ndarray): return a - a = np.vectorize(lambda v: v.value, otypes='O')(a) + a = np.vectorize(lambda v: v.value, otypes="O")(a) return type(self)(a, check=False) def trace(self, *args, **kwargs): @@ -1347,7 +1347,7 @@ def transpose(self, *axes): a = self.value.transpose(*axes) return type(self)(a, check=False) - def swapaxes(self, axis1, axis2): # view -- no copy + def swapaxes(self, axis1, axis2): # view -- no copy a = self.value.swapaxes(axis1, axis2) return type(self)(a, check=False) @@ -1357,11 +1357,9 @@ def T(self): class PrimeFieldArray(FiniteFieldArray): - _mix_types = int if np: - _mix_types = (int, np.int64, np.int32, np.int16, np.int8, - np.uint64, np.uint32, np.uint16, np.uint8) + _mix_types = (int, np.int64, np.int32, np.int16, np.int8, np.uint64, np.uint32, np.uint16, np.uint8) # TODO: consider use of np.integer @classmethod @@ -1385,7 +1383,7 @@ def __abs__(self): def signed_(self): """Return signed integer representation, symmetric around zero.""" p = self.field.modulus - f = np.vectorize(lambda a: a - p if a > p>>1 else a, otypes='O') + f = np.vectorize(lambda a: a - p if a > p >> 1 else a, otypes="O") return f(self.value) # TODO: check following alternative, issue with np.where result containing floats # return self.value - np.where(self.value > p>>1, p, 0) @@ -1399,7 +1397,7 @@ def _pow(cls, a, b): # NB: exponents assumed to be integer(s) """Exponentiation.""" p = cls.field.modulus powmod = gmpy2.powmod - f = np.vectorize(lambda a, b: int(powmod(a, b, p)), otypes='O') + f = np.vectorize(lambda a, b: int(powmod(a, b, p)), otypes="O") return f(a, b) @classmethod @@ -1407,7 +1405,7 @@ def _reciprocal(cls, a): """Multiplicative inverse.""" p = cls.field.modulus invert = gmpy2.invert - f = np.vectorize(lambda a: int(invert(a, p)), otypes='O') + f = np.vectorize(lambda a: int(invert(a, p)), otypes="O") return f(a) # NB: otypes='O' ensures entries of a are Python int (before f is applied) @classmethod @@ -1415,42 +1413,43 @@ def _sqrt(cls, a, INV=False): """Modular (inverse) square roots.""" p = cls.field.modulus if INV and (a == 0).any(): - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") if p == 2: return a.copy() # TODO: check use of copy (here: copy returned in all cases for p) - if p&3 == 3: + if p & 3 == 3: if INV: - p4 = (p*3 - 5) >> 2 # a**p4 == a**(-1/2) == 1/sqrt(a) mod p + p4 = (p * 3 - 5) >> 2 # a**p4 == a**(-1/2) == 1/sqrt(a) mod p else: - p4 = (p+1) >> 2 + p4 = (p + 1) >> 2 p = gmpy2.mpz(p) p4 = gmpy2.mpz(p4) - W = int(os.getenv('MPYC_MAXWORKERS')) + W = int(os.getenv("MPYC_MAXWORKERS")) if W == 0: powmod = gmpy2.powmod - return np.vectorize(lambda a: int(powmod(a, p4, p)), otypes='O')(a) + return np.vectorize(lambda a: int(powmod(a, p4, p)), otypes="O")(a) # Experimental use of W worker threads. # Using gmpy2's new function powmod_base_list(), which releases the GIL. # Example: "python np_lpsolver.py -i6 -M3 -W2" about 1.4x faster than for W=0. - from gmpy2 import powmod_base_list # NB: requires gmpy2 >= 2.1.3 import concurrent.futures + + from gmpy2 import powmod_base_list # NB: requires gmpy2 >= 2.1.3 + n = a.size s = a.flat with concurrent.futures.ThreadPoolExecutor(max_workers=W) as executor: - tasks = {executor.submit(powmod_base_list, s[i*n//W:(i+1)*n//W], p4, p): i - for i in range(W)} - s = np.empty(n, dtype='O') + tasks = {executor.submit(powmod_base_list, s[i * n // W : (i + 1) * n // W], p4, p): i for i in range(W)} + s = np.empty(n, dtype="O") for task in concurrent.futures.as_completed(tasks): i = tasks[task] - s[i*n//W:(i+1)*n//W] = task.result() - return np.vectorize(int, otypes='O')(s).reshape(a.shape) + s[i * n // W : (i + 1) * n // W] = task.result() + return np.vectorize(int, otypes="O")(s).reshape(a.shape) _sqrt = cls.field._sqrt - return np.vectorize(lambda a: _sqrt(a, INV=INV), otypes='O')(a) + return np.vectorize(lambda a: _sqrt(a, INV=INV), otypes="O")(a) @classmethod def _is_sqr(cls, a): @@ -1462,22 +1461,20 @@ def _is_sqr(cls, a): return np.vectorize(lambda a: legendre(a, p) != -1, otypes=[bool])(a) def __repr__(self): - return f'{self.intarray(self)}' + return f"{self.intarray(self)}" class ExtensionFieldArray(FiniteFieldArray): - _mix_types = (int, gfpx.Polynomial) if np: - _mix_types += (np.int64, np.int32, np.int16, np.int8, - np.uint64, np.uint32, np.uint16, np.uint8) + _mix_types += (np.int64, np.int32, np.int16, np.int8, np.uint64, np.uint32, np.uint16, np.uint8) @classmethod def _pow(cls, a, b): # NB: exponents assumed to be integer(s) """Exponentiation.""" modulus = cls.field.modulus powmod = type(modulus).powmod - f = np.vectorize(lambda a, n: powmod(a, n, modulus), otypes='O') + f = np.vectorize(lambda a, n: powmod(a, n, modulus), otypes="O") return f(a, b) @classmethod @@ -1485,18 +1482,18 @@ def _reciprocal(cls, a): """Multiplicative inverse.""" modulus = cls.field.modulus invert = type(modulus).invert - f = np.vectorize(lambda a: invert(a, modulus), otypes='O') + f = np.vectorize(lambda a: invert(a, modulus), otypes="O") return f(a) @classmethod def _is_sqr(cls, a): q = cls.field.order - if q%2 == 0: + if q % 2 == 0: return np.full(a.shape, True, dtype=bool) modulus = cls.field.modulus powmod = type(modulus).powmod - q2 = (q-1) >> 1 + q2 = (q - 1) >> 1 minus_one = [type(modulus).p - 1] f = np.vectorize(lambda a: powmod(a, q2, modulus) != minus_one, otypes=[bool]) return f(a) @@ -1508,36 +1505,34 @@ def _sqrt(cls, a, INV=False): modulus = cls.field.modulus poly = type(modulus) if INV and (a == poly(0)).any(): - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") - if q%2 == 0: + if q % 2 == 0: q2 = q >> 1 if INV: q2 -= 1 powmod = poly.powmod - return np.vectorize(lambda a: powmod(a, q2, modulus), otypes='O')(a) + return np.vectorize(lambda a: powmod(a, q2, modulus), otypes="O")(a) - if q&3 == 3: + if q & 3 == 3: if INV: - q4 = (q*3 - 5) >> 2 # a**q4 == a**(-1/2) == 1/sqrt(a) in GF(q) + q4 = (q * 3 - 5) >> 2 # a**q4 == a**(-1/2) == 1/sqrt(a) in GF(q) else: - q4 = (q+1) >> 2 + q4 = (q + 1) >> 2 powmod = poly.powmod - return np.vectorize(lambda a: powmod(a, q4, modulus), otypes='O')(a) + return np.vectorize(lambda a: powmod(a, q4, modulus), otypes="O")(a) _sqrt = cls.field._sqrt - return np.vectorize(lambda a: _sqrt(a, INV=INV), otypes='O')(a) + return np.vectorize(lambda a: _sqrt(a, INV=INV), otypes="O")(a) def __repr__(self): - return f'{self.value}' + return f"{self.value}" class BinaryFieldArray(ExtensionFieldArray): - _mix_types = (int, gfpx.BinaryPolynomial) if np: - _mix_types += (np.int64, np.int32, np.int16, np.int8, - np.uint64, np.uint32, np.uint16, np.uint8) + _mix_types += (np.int64, np.int32, np.int16, np.int8, np.uint64, np.uint32, np.uint16, np.uint8) @classmethod def _is_sqr(cls, a): @@ -1548,10 +1543,10 @@ def _sqrt(cls, a, INV=False): modulus = cls.field.modulus poly = type(modulus) if INV and (a == poly(0)).any(): - raise ZeroDivisionError('no inverse sqrt of 0') + raise ZeroDivisionError("no inverse sqrt of 0") powmod = poly.powmod q2 = cls.field.order >> 1 if INV: q2 -= 1 - return np.vectorize(lambda a: powmod(a, q2, modulus), otypes='O')(a) + return np.vectorize(lambda a: powmod(a, q2, modulus), otypes="O")(a) diff --git a/mpyc-web-py/mpycweb/proxy.py b/mpyc-web-py/mpycweb/proxy.py index 5d81d35a..e218e14c 100644 --- a/mpyc-web-py/mpycweb/proxy.py +++ b/mpyc-web-py/mpycweb/proxy.py @@ -77,11 +77,11 @@ async def create_connection( self.transports[pid] = t return t, p - @stats.acc(lambda self, pid, message: stats.sent_to(pid, message)) + # @stats.acc(lambda self, pid, message: stats.sent_to(pid, message)) def send_ready_message(self, pid: int, message: str): self.async_proxy.send("proxy:js:mpc:msg:ready", pid, message) - @stats.acc(lambda self, pid, message: stats.received_from(pid, message)) + # @stats.acc(lambda self, pid, message: stats.received_from(pid, message)) def on_ready_message(self, pid: int, message: str): """ Handle a 'ready' message from a peer. @@ -98,12 +98,13 @@ def on_ready_message(self, pid: int, message: str): return self.transports[pid].on_ready_message(message) - @stats.acc(lambda self, pid, message: stats.sent_to(pid, message) | stats.time()) + # @stats.acc(lambda self, pid, message: stats.sent_to(pid, message)) + # @stats.time() def send_runtime_message(self, pid: int, message: bytes): - # add timestamp self.async_proxy.send("proxy:js:mpc:msg:runtime", pid, message) - @stats.acc(lambda self, pid, message, ts: stats.received_from(pid, message) | stats.time() | stats.latency(ts)) + # @stats.acc(lambda self, pid, message, ts: stats.received_from(pid, message) | stats.latency(ts)) + # @stats.time() def on_runtime_message(self, pid: int, message: JsProxy, ts) -> None: """ Handle a runtime message from a peer. diff --git a/mpyc-web-starter/index.html b/mpyc-web-starter/index.html index 1f94e5b6..aee891c9 100644 --- a/mpyc-web-starter/index.html +++ b/mpyc-web-starter/index.html @@ -10,7 +10,7 @@