From ad00805ce23d833edd5c9abef9739b153aaeef5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=A0orn?= Date: Fri, 30 Jun 2023 22:18:18 +0200 Subject: [PATCH] Rewrite of Profiling --- README.md | 75 ++++++++++++++++---------------- index.html | 83 ++++++++++++++++++------------------ parse.js | 39 ++++++++++++----- pdf/index_for_pdf.html | 4 +- pdf/index_for_pdf_print.html | 2 +- 5 files changed, 109 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 5a072d8..7e36166 100644 --- a/README.md +++ b/README.md @@ -1935,14 +1935,14 @@ with .begin(): ... # Exits the block with commit or ``` ```text -+------------+--------------+-----------+----------------------------------+ -| Dialect | pip3 install | import | Dependencies | -+------------+--------------+-----------+----------------------------------+ -| mysql | mysqlclient | MySQLdb | www.pypi.org/project/mysqlclient | -| postgresql | psycopg2 | psycopg2 | www.pypi.org/project/psycopg2 | -| mssql | pyodbc | pyodbc | www.pypi.org/project/pyodbc | -| oracle | oracledb | oracledb | www.pypi.org/project/oracledb | -+------------+--------------+-----------+----------------------------------+ ++------------+--------------+----------+----------------------------------+ +| Dialect | pip3 install | import | Dependencies | ++------------+--------------+----------+----------------------------------+ +| mysql | mysqlclient | MySQLdb | www.pypi.org/project/mysqlclient | +| postgresql | psycopg2 | psycopg2 | www.pypi.org/project/psycopg2 | +| mssql | pyodbc | pyodbc | www.pypi.org/project/pyodbc | +| oracle | oracledb | oracledb | www.pypi.org/project/oracledb | ++------------+--------------+----------+----------------------------------+ ``` @@ -2591,49 +2591,48 @@ duration_in_seconds = perf_counter() - start_time ### Timing a Snippet ```python >>> from timeit import timeit ->>> timeit("''.join(str(i) for i in range(100))", -... number=10000, globals=globals(), setup='pass') -0.34986 +>>> timeit('list(range(10000))', number=1000, globals=globals(), setup='pass') +0.19373 ``` ### Profiling by Line -```python -# $ pip3 install line_profiler memory_profiler +```text +$ pip3 install line_profiler +$ echo " @profile def main(): - a = [*range(10000)] - b = {*range(10000)} -main() -``` - -```text + a = list(range(10000)) + b = set(range(10000)) +main()" > test.py $ kernprof -lv test.py Line # Hits Time Per Hit % Time Line Contents ======================================================= 1 @profile 2 def main(): - 3 1 955.0 955.0 43.7 a = [*range(10000)] - 4 1 1231.0 1231.0 56.3 b = {*range(10000)} - -$ python3 -m memory_profiler test.py -Line # Mem usage Increment Line Contents -======================================================= - 1 37.668 MiB 37.668 MiB @profile - 2 def main(): - 3 38.012 MiB 0.344 MiB a = [*range(10000)] - 4 38.477 MiB 0.465 MiB b = {*range(10000)} + 3 1 219.0 219.0 31.1 a = list(range(10000)) + 4 1 487.0 487.0 68.9 b = set(range(10000)) ``` -### Call Graph -#### Generates a PNG image of the call graph with highlighted bottlenecks: -```python -# $ pip3 install pycallgraph2; apt/brew install graphviz -import pycallgraph2 as cg, datetime +### Call and Flame Graphs +```bash +$ pip3 install gprof2dot snakeviz +$ apt/brew install graphviz +$ python3 -m cProfile -o test.prof test.py +$ gprof2dot -f pstats test.prof | dot -Tpng -o test.png; xdg-open/open test.png +$ snakeviz test.prof +``` -filename = f'profile-{datetime.datetime.now():%Y%m%d_%H%M%S}.png' -drawer = cg.output.GraphvizOutput(output_file=filename) -with cg.PyCallGraph(drawer): - +### Sampling and Memory Profilers +```text ++--------------+-------------------------------+------------+----------+------+ +| pip3 install | How to run | Target | Type | Live | ++--------------+-------------------------------+------------+----------+------+ +| py-spy | py-spy top -- python3 test.py | CPU | Sampling | Yes | +| pyinstrument | pyinstrument test.py | CPU | Sampling | No | +| scalene | scalene test.py | CPU+Memory | Sampling | No | +| memray | memray run --live test.py | Memory | Tracing | Yes | +| filprofiler | fil-profile run test.py | Memory | Tracing | No | ++--------------+-------------------------------+------------+----------+------+ ``` diff --git a/index.html b/index.html index d11dfc4..4888989 100644 --- a/index.html +++ b/index.html @@ -54,7 +54,7 @@
- +
@@ -1609,14 +1609,14 @@ CompletedProcess(args=['bc', with <conn>.begin(): ... # Exits the block with commit or rollback. -
┏━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
-┃ Dialect    │ pip3 install │ import    │ Dependencies                     ┃
-┠────────────┼──────────────┼───────────┼──────────────────────────────────┨
-┃ mysql      │ mysqlclient  │ MySQLdb   │ www.pypi.org/project/mysqlclient ┃
-┃ postgresql │ psycopg2     │ psycopg2  │ www.pypi.org/project/psycopg2    ┃
-┃ mssql      │ pyodbc       │ pyodbc    │ www.pypi.org/project/pyodbc      ┃
-┃ oracle     │ oracledb     │ oracledb  │ www.pypi.org/project/oracledb    ┃
-┗━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
┏━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ Dialect    │ pip3 install │  import  │           Dependencies           ┃
+┠────────────┼──────────────┼──────────┼──────────────────────────────────┨
+┃ mysql      │ mysqlclient  │ MySQLdb  │ www.pypi.org/project/mysqlclient ┃
+┃ postgresql │ psycopg2     │ psycopg2 │ www.pypi.org/project/psycopg2    ┃
+┃ mssql      │ pyodbc       │ pyodbc   │ www.pypi.org/project/pyodbc      ┃
+┃ oracle     │ oracledb     │ oracledb │ www.pypi.org/project/oracledb    ┃
+┗━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
 

 
#Bytes
Bytes object is an immutable sequence of single bytes. Mutable version is called bytearray.
<bytes> = b'<str>'                          # Only accepts ASCII characters and \x00-\xff.
 <int>   = <bytes>[<index>]                  # Returns an int in range from 0 to 255.
@@ -2124,44 +2124,43 @@ duration_in_seconds = perf_counter() - start_time
 

 
 
Timing a Snippet
>>> from timeit import timeit
->>> timeit("''.join(str(i) for i in range(100))",
-...        number=10000, globals=globals(), setup='pass')
-0.34986
-

-
-
Profiling by Line
# $ pip3 install line_profiler memory_profiler
-@profile
-def main():
-    a = [*range(10000)]
-    b = {*range(10000)}
-main()
-

-
-
$ kernprof -lv test.py
+>>> timeit('list(range(10000))', number=1000, globals=globals(), setup='pass')
+0.19373
+

+
+
Profiling by Line
$ pip3 install line_profiler
+$ echo "
+@profile
+def main():
+    a = list(range(10000))
+    b = set(range(10000))
+main()" > test.py
+$ kernprof -lv test.py
 Line #   Hits     Time  Per Hit   % Time  Line Contents
 =======================================================
      1                                    @profile
      2                                    def main():
-     3      1    955.0    955.0     43.7      a = [*range(10000)]
-     4      1   1231.0   1231.0     56.3      b = {*range(10000)}
-
-$ python3 -m memory_profiler test.py
-Line #         Mem usage      Increment   Line Contents
-=======================================================
-     1        37.668 MiB     37.668 MiB   @profile
-     2                                    def main():
-     3        38.012 MiB      0.344 MiB       a = [*range(10000)]
-     4        38.477 MiB      0.465 MiB       b = {*range(10000)}
-

-
Call Graph
Generates a PNG image of the call graph with highlighted bottlenecks:
# $ pip3 install pycallgraph2; apt/brew install graphviz
-import pycallgraph2 as cg, datetime
+     3      1    219.0    219.0     31.1      a = list(range(10000))
+     4      1    487.0    487.0     68.9      b = set(range(10000))
+

 
-filename = f'profile-{datetime.datetime.now():%Y%m%d_%H%M%S}.png'
-drawer = cg.output.GraphvizOutput(output_file=filename)
-with cg.PyCallGraph(drawer):
-    <code_to_be_profiled>
-
+

Call and Flame Graphs

$ pip3 install gprof2dot snakeviz
+$ apt/brew install graphviz
+$ python3 -m cProfile -o test.prof test.py
+$ gprof2dot -f pstats test.prof | dot -Tpng -o test.png; xdg-open/open test.png
+$ snakeviz test.prof
+
+

Sampling and Memory Profilers

┏━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━┓
+┃ pip3 install │          How to run           │   Target   │   Type   │ Live ┃
+┠──────────────┼───────────────────────────────┼────────────┼──────────┼──────┨
+┃ py-spy       │ py-spy top -- python3 test.py │    CPU     │ Sampling │ Yes  ┃
+┃ pyinstrument │ pyinstrument test.py          │    CPU     │ Sampling │ No   ┃
+┃ scalene      │ scalene test.py               │ CPU+Memory │ Sampling │ No   ┃
+┃ memray       │ memray run --live test.py     │   Memory   │ Tracing  │ Yes  ┃
+┃ filprofiler  │ fil-profile run test.py       │   Memory   │ Tracing  │ No   ┃
+┗━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━┛
+

#NumPy

Array manipulation mini-language. It can run up to one hundred times faster than the equivalent Python code. An even faster alternative that runs on a GPU is called CuPy.

# $ pip3 install numpy
 import numpy as np
@@ -2934,7 +2933,7 @@ $ pyinstaller script.py --add-data '<path>:.'  
  
 
   
 
diff --git a/parse.js b/parse.js
index 93d9630..d7f978d 100755
--- a/parse.js
+++ b/parse.js
@@ -460,19 +460,19 @@ const DIAGRAM_9_B =
   "┗━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━┛\n";
 
 const DIAGRAM_95_A =
-  "+------------+--------------+-----------+----------------------------------+\n" +
-  "| Dialect    | pip3 install | import    | Dependencies                     |\n" +
-  "+------------+--------------+-----------+----------------------------------+\n";
+  '+------------+--------------+----------+----------------------------------+\n' +
+  '| Dialect    | pip3 install |  import  |           Dependencies           |\n' +
+  '+------------+--------------+----------+----------------------------------+\n';
 
 const DIAGRAM_95_B =
-  "┏━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n" +
-  "┃ Dialect    │ pip3 install │ import    │ Dependencies                     ┃\n" +
-  "┠────────────┼──────────────┼───────────┼──────────────────────────────────┨\n" +
-  "┃ mysql      │ mysqlclient  │ MySQLdb   │ www.pypi.org/project/mysqlclient ┃\n" +
-  "┃ postgresql │ psycopg2     │ psycopg2  │ www.pypi.org/project/psycopg2    ┃\n" +
-  "┃ mssql      │ pyodbc       │ pyodbc    │ www.pypi.org/project/pyodbc      ┃\n" +
-  "┃ oracle     │ oracledb     │ oracledb  │ www.pypi.org/project/oracledb    ┃\n" +
-  "┗━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\n";
+  '┏━━━━━━━━━━━━┯━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n' +
+  '┃ Dialect    │ pip3 install │  import  │           Dependencies           ┃\n' +
+  '┠────────────┼──────────────┼──────────┼──────────────────────────────────┨\n' +
+  '┃ mysql      │ mysqlclient  │ MySQLdb  │ www.pypi.org/project/mysqlclient ┃\n' +
+  '┃ postgresql │ psycopg2     │ psycopg2 │ www.pypi.org/project/psycopg2    ┃\n' +
+  '┃ mssql      │ pyodbc       │ pyodbc   │ www.pypi.org/project/pyodbc      ┃\n' +
+  '┃ oracle     │ oracledb     │ oracledb │ www.pypi.org/project/oracledb    ┃\n' +
+  '┗━━━━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛\n';
 
 const DIAGRAM_10_A =
   '+-------------+-------------+\n' +
@@ -508,6 +508,22 @@ const DIAGRAM_11_B =
   '┃     str     │             ┃\n' +
   '┗━━━━━━━━━━━━━┷━━━━━━━━━━━━━┛\n';
 
+const DIAGRAM_115_A =
+  '+--------------+-------------------------------+------------+----------+------+\n' +
+  '| pip3 install |          How to run           |   Target   |   Type   | Live |\n' +
+  '+--------------+-------------------------------+------------+----------+------+\n';
+
+const DIAGRAM_115_B =
+  '┏━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━┓\n' +
+  '┃ pip3 install │          How to run           │   Target   │   Type   │ Live ┃\n' +
+  '┠──────────────┼───────────────────────────────┼────────────┼──────────┼──────┨\n' +
+  '┃ py-spy       │ py-spy top -- python3 test.py │    CPU     │ Sampling │ Yes  ┃\n' +
+  '┃ pyinstrument │ pyinstrument test.py          │    CPU     │ Sampling │ No   ┃\n' +
+  '┃ scalene      │ scalene test.py               │ CPU+Memory │ Sampling │ No   ┃\n' +
+  '┃ memray       │ memray run --live test.py     │   Memory   │ Tracing  │ Yes  ┃\n' +
+  '┃ filprofiler  │ fil-profile run test.py       │   Memory   │ Tracing  │ No   ┃\n' +
+  '┗━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━┛\n';
+
 const DIAGRAM_12_A =
   '+-----------+-----------+------+-----------+\n' +
   '| sampwidth |    min    | zero |    max    |\n' +
@@ -740,6 +756,7 @@ function updateDiagrams() {
   $(`code:contains(${DIAGRAM_95_A})`).html(DIAGRAM_95_B);
   $(`code:contains(${DIAGRAM_10_A})`).html(DIAGRAM_10_B);
   $(`code:contains(${DIAGRAM_11_A})`).html(DIAGRAM_11_B);
+  $(`code:contains(${DIAGRAM_115_A})`).html(DIAGRAM_115_B);
   $(`code:contains(${DIAGRAM_12_A})`).html(DIAGRAM_12_B).removeClass("text").removeClass("language-text").addClass("python");
   $(`code:contains(${DIAGRAM_13_A})`).html(DIAGRAM_13_B).removeClass("text").removeClass("language-text").addClass("python");
   $(`code:contains(${DIAGRAM_14_A})`).parent().remove();
diff --git a/pdf/index_for_pdf.html b/pdf/index_for_pdf.html
index 70fbdb6..81aaa34 100644
--- a/pdf/index_for_pdf.html
+++ b/pdf/index_for_pdf.html
@@ -51,7 +51,7 @@
 filter function, 11

 flask library, 36

 floats, 4, 6, 7

-format, 6-7, 37

+format, 6-7

 functools module, 11, 12, 13, 16

 futures, 30  

 
G

@@ -103,7 +103,7 @@
 pillow library, 39-40

 plotting, 34, 46, 47-48

 print function, 22

-profiling, 36-37

+profiling, 36-37

 progress bar, 34

 property decorator, 15

 pygame library, 42-43  

diff --git a/pdf/index_for_pdf_print.html b/pdf/index_for_pdf_print.html
index e65c322..001ce8f 100644
--- a/pdf/index_for_pdf_print.html
+++ b/pdf/index_for_pdf_print.html
@@ -51,7 +51,7 @@
 filter function, 11

 flask library, 36

 floats, 4, 6, 7

-format, 6-7, 37

+format, 6-7

 functools module, 11, 12, 13, 16

 futures, 30  

 
G